This commit is contained in:
2024-04-30 15:43:48 +08:00
parent e526e1edff
commit 3a8b5254c3
20 changed files with 1178 additions and 381 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/** /**
* 深圳公众号成交数据录入 * 深圳公众号成交数据录入
*/ */
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1) def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -75,18 +75,22 @@ for(def d : datas){
map.area = d['F'].toString().toDouble() map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble()) map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000) if(d['I']){
def trade_date = d['J'] map.sell_price = (int)(d['I'].toString().toDouble()*10000)
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
} }
// def trade_date = d['J']
// if(trade_date){
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
// }else{
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
// }
map.source = "深房家族" map.source = "深房家族"
map.create_date = new Date() map.create_date = new Date()
map.m_date = new Date() map.m_date = new Date()
println map
DEAL_PRICE_COLL.save(map) DEAL_PRICE_COLL.save(map)
} }
@@ -69,51 +69,51 @@ class BigdataDealBatch extends BaseBatch{
addDealPriceMatch(city); addDealPriceMatch(city);
} }
def rows = [] // def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] // def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0 // def index = 0
for(def city : citys){ // for(def city : citys){
index ++ // index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:] // def ppeDists = [:]
//
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) // def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size() // println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend) // def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){ // if(city=="上海"){
def youDataTrade = getYouDaData(statMap) // def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) // def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas) // rows.addAll(youdaDatas)
println city +" "+youdaDatas.size() // println city +" "+youdaDatas.size()
} // }
//
def trades = getCityData(city, statMap, ppeDists) // def trades = getCityData(city, statMap, ppeDists)
if(trades){ // if(trades){
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) // def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets) // rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size() // println "【${index}】"+city +" 成交共:"+rets.size()
//
def cityDatas = [] // def cityDatas = []
cityDatas << headers // cityDatas << headers
cityDatas.addAll(rows) // cityDatas.addAll(rows)
} // }
//
} // }
//
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" // def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){ // if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" // fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
} // }
//
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); // FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size() // println "txt 生成 " + rows.size()
//
// delPgData() //// delPgData()
saveToPgSql(rows) // saveToPgSql(rows)
//
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") //// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas) //// saveToPgSql2(datas)
} }
def readTxt(def fileName){ def readTxt(def fileName){
@@ -388,19 +388,21 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){ def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
def citys = ["469005"]//CcbCommonUtils.citys_all // def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = [] def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet() def set = new HashSet()
for(def cityid : citys) { for(def city : citys) {
def cityid = cityCodeDatas.get(city)
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
println cityid +" "+rets.size() println city +" "+rets.size()
if(rets){ if(rets){
set.add(cityid) set.add(city)
} }
for(def m : rets){ for(def m : rets){
// def cityId = cityCodeDatas.get(city)
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql) // sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
@@ -409,7 +411,7 @@ class DealPriceDataProcess {
total = r.count total = r.count
}) })
if(total>0){ if(total>0){
println cityid +" "+m.commid+" (按价格)删除:"+total println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql) pgsqlInstance.executeUpdate(sql)
}else{ }else{
@@ -419,7 +421,7 @@ class DealPriceDataProcess {
total = r.count total = r.count
}) })
if(total>0){ if(total>0){
println cityId +" "+m.commid+" (按小区)删除:"+total println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql) pgsqlInstance.executeUpdate(sql)
} }
@@ -450,7 +452,7 @@ class DealPriceDataProcess {
} }
def findPeGteThirdParty2(cityid, yearmonth){ def findPeGteThirdParty2(cityid, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20" def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
def rets = [] def rets = []
pgsqlInstance.eachRow(sql,{r-> pgsqlInstance.eachRow(sql,{r->
def map = [:] def map = [:]
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-10") def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
@@ -1289,70 +1289,66 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx" def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls"
//
//ExcelUtils.multiThreadingModify(fileName, {d-> ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){ if(d['C']){
//// def region = RegionData.formatRegion(d['B'], d['C']) // def region = RegionData.formatRegion(d['B'], d['C'])
//// if(region){ // if(region){
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//// if(dist){
//// d['G'] = dist.region
//// d['H'] = dist._id.toString()
//// }
//// }
// if(!d['I']){
// def dist = CommonDatautils.findDistrictById(d['C'])
// if(dist){ // if(dist){
// d['F'] = dist.region // d['G'] = dist.region
//// d['H'] = dist.name // d['H'] = dist._id.toString()
// d['G'] = dist.status
//// if(d['A']=="天门" || d['A']=="迪庆"){
//// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
//// d['K'] = val.toInteger()
//// }
// } // }
// } // }
// } if(!d['L']){
//},0) def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" d['L'] = dist.region
ExcelUtils.multiThreadingModify(fileName, { d -> d['M'] = dist.name
if(d['B'] && !d['D']){ d['N'] = dist.status
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
if(jsonObject != null && jsonObject.credit>0) {
d['D'] = jsonObject.get("city")
d['E'] = jsonObject.get("district")
d['F'] = jsonObject.get("name")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['G'] = dbo.status
d['H'] = dbo.name_cat
d['I'] = dbo.estate_type2
d['J'] = "接口匹配"
} }
} }
}
},0)
// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district") //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
// if(!objs){ //ExcelUtils.multiThreadingModify(fileName, { d ->
// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district") // if(d['B'] && !d['D']){
// } // def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
// if(objs && objs.size()==1){ // if(jsonObject != null && jsonObject.credit>0) {
// def o = objs[0] // d['D'] = jsonObject.get("city")
// if(o){ // d['E'] = jsonObject.get("district")
// d['D'] = o.city // d['F'] = jsonObject.get("name")
// d['E'] = o.region // def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
// d['F'] = o.name // if(dbo){
// d['G'] = o.status // d['G'] = dbo.status
// d['H'] = o.name_cat // d['H'] = dbo.name_cat
// d['I'] = o.estate_type2 // d['I'] = dbo.estate_type2
// d['J'] = "别名" // d['J'] = "接口匹配"
// } // }
// } // }
//
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
} //// if(!objs){
}) //// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
//// }
//// if(objs && objs.size()==1){
//// def o = objs[0]
//// if(o){
//// d['D'] = o.city
//// d['E'] = o.region
//// d['F'] = o.name
//// d['G'] = o.status
//// d['H'] = o.name_cat
//// d['I'] = o.estate_type2
//// d['J'] = "别名"
//// }
//// }
//
//
// }
//})
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx" //def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d -> //ExcelUtils.multiThreadingModify(fileName, { d ->
@@ -1,6 +1,7 @@
package com.fangjia.script.commData package com.fangjia.script.commData
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.httpclient.UserAgents import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.InsidePolygon
@@ -8,11 +9,16 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import groovy.json.JsonSlurper import groovy.json.JsonSlurper
import net.sf.json.JSONArray import net.sf.json.JSONArray
import net.sf.json.JSONObject import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.bson.types.ObjectId import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest; import com.fangjia.core.util.HttpRequest;
@@ -110,6 +116,7 @@ class CommonDatautils {
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
} }
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static DSCollection getHouseDataStoreConnection(){ public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) { if(house_dataStoreConnection != null) {
@@ -1459,6 +1466,28 @@ class CommonDatautils {
return obj; return obj;
} }
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('status',0)
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, [name]);
q.put("mapping_alias", query);
def dbcursor = SEAWEED_COLL.find(q);
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
static def findSeaweeds(city, cat, name_cat){ static def findSeaweeds(city, cat, name_cat){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put('city',city) q.put('city',city)
@@ -4601,4 +4630,6 @@ class CommonDatautils {
} }
@@ -0,0 +1,3 @@
package com.fangjia.script.fuzhou
@@ -29,11 +29,11 @@ class FzDataResultStat {
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
// task.priceMarkupRun(); task.priceMarkupRun();
// task.regionPriceTrendRun() // task.regionPriceTrendRun()
//// task.margeExcel() //// task.margeExcel()
// task.saveCsvData() // task.saveCsvData()
task.testNewPrice() // task.testNewPrice()
} }
def margeExcel(){ def margeExcel(){
@@ -58,7 +58,7 @@ class FzDataResultStat {
} }
def priceMarkupRun(){ def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15"); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) // List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) // def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
@@ -82,7 +82,7 @@ class FzDataResultStat {
} }
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) // List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth) def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
def map2 = [:] def map2 = [:]
for(def obj : list2) { for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name def key = obj.city + "##" + obj.region + "##" + obj.name
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
@@ -32,7 +33,9 @@ class GXDdistrictMatch {
// task.updateDistStatus()//恢复name_cat不正常数据 // task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区 // task.saveToDB() //保存新增小区
task.restoreDist()//恢复小区status=0 // task.restoreDist()//恢复小区status=0
task.runMatch()
} }
def addOldMatch(){ def addOldMatch(){
@@ -133,6 +136,22 @@ class GXDdistrictMatch {
}) })
} }
def runMatch(){
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
def city = "福州"
ExcelUtils.modify(fileName, { d ->
println d['C']
if(d['B'] && d['A']){
def region = RegionData.formatRegion(city, d['B'])
if (region) {
d = match(city, region, d)
} else {
println d['A'] + "#" + d['E'] + " 区域无法格式化"
}
}
})
}
def updateDistStatus(){ def updateDistStatus(){
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx" def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
def index = 0 def index = 0
@@ -187,27 +206,32 @@ class GXDdistrictMatch {
} }
def saveToDB(){ def saveToDB(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" // def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
ExcelUtils.modify(fileName, { d -> ExcelUtils.modify(fileName, { d ->
if(d['U'] && d['AG']!='过滤'){ if(d['A']){
def udata = new BasicDBObject() def udata = new BasicDBObject()
udata.put("city", d['AD']) udata.put("city", d['A'])
udata.put("region", d['AE']) udata.put("region", d['B'])
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name) udata.put("name", name)
def alias = [] def alias = []
if(name!=d['C']){ if(d['G'] && name!=d['G']){
alias << d['C'] alias << d['G']
}
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
alias << d['AI']
} }
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){ if(alias){
udata.put("alias", alias) udata.put("alias", alias)
} }
udata.put("address", d['AF']) def addr = AddressUtil.formatAddress(d['A'], d['D'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0") udata.put("status", "0")
udata.put("name_cat", "0") udata.put("name_cat", "0")
udata.put("cat", "district") udata.put("cat", "district")
@@ -215,11 +239,11 @@ class GXDdistrictMatch {
def estateType2 = "普通住宅" def estateType2 = "普通住宅"
def type = "" def type = ""
if(d['AH']){ // if(d['AH']){
type = d['AH'] // type = d['AH']
}else{ // }else{
type = d['Y'] // type = d['Y']
} // }
if(type){ if(type){
if(type.contains("别墅")){ if(type.contains("别墅")){
estateType2 = "别墅" estateType2 = "别墅"
@@ -228,12 +252,12 @@ class GXDdistrictMatch {
} }
} }
udata.put("estateType2", estateType2) udata.put("estateType2", estateType2)
if(d['AA']){ if(d['F']){
Long lng = d['AA']*10000000000 Long lng = d['F']*10000000000
udata.put("lng", lng) udata.put("lng", lng)
} }
if(d['AB']){ if(d['E']){
Long lat = d['AB'] * 10000000000 Long lat = d['E'] * 10000000000
udata.put("lat", lat) udata.put("lat", lat)
} }
@@ -265,7 +289,7 @@ class GXDdistrictMatch {
if(!d['U']){ if(!d['U']){
d = matchApi(city, region, d, "property", false) d = matchApi(city, region, d, "property", false)
if(!d['U']){ if(!d['U']){
d = matchApi(city, region, d, null, true) d = matchApi(city, region, d, null, false)
} }
} }
@@ -285,14 +309,18 @@ class GXDdistrictMatch {
} }
def matchApi(city, region, d, cat, useMap){ def matchApi(city, region, d, cat, useMap){
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) def jsonObject = null
if(!jsonObject){ if(!jsonObject && d['H']){
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
}
if(!jsonObject && d['C']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap) jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
if(!jsonObject && d['D']){ if(!jsonObject && d['D']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap) jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
} }
} }
if(jsonObject != null && jsonObject.credit>0) { if(jsonObject != null) {
d['U'] = "接口匹配" d['U'] = "接口匹配"
d['V'] = jsonObject.get("city") d['V'] = jsonObject.get("city")
d['W'] = jsonObject.get("district") d['W'] = jsonObject.get("district")
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class TempExportData {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new TempExportData()
task.run()
}
def run() {
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
for (def city : citys) {
def cityRets = getResults(city)
rets.addAll(cityRets)
}
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
}
def getResults(city) {
def rets = []
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
final AtomicInteger count_ = new AtomicInteger(0)
for (def i = 0; i < dists.size(); i++) {
def d = dists[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try {
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
if (objs) {
def ret = []
ret << d._id.toString()
ret << d.city
ret << d.region
ret << d.name
ret << d.address
def str = ""
for (def o : objs) {
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
}
ret << str
ret << objs.size()
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
println(city +" "+rets.size())
return rets;
}
}
@@ -0,0 +1,544 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
*/
class TownMiddleProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new TownMiddleProcess();
task.run()
}
def run(){
//创建省份
// createProvince()
//创建城市
// createCity()
//创建区域
// createRegion()
//创建乡镇街道
createTown()
}
def createProvince(){
def cat = "省"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
tjjMaps.each {key, val->
def udata = new BasicDBObject()
udata.put("province", key)
udata.put("tjj_province_code", val)
udata.put("mzb_province_code", mzbMaps.get(key))
udata.put("tjj_source_year",2023)
udata.put("mzb_source_year",2023)
lastRets << udata
}
if(diffKeys){
diffKeys.each {k->
def udata = new BasicDBObject()
udata.put("province", k)
if(tjjMaps.get(k)){
udata.put("tjj_province_code", tjjMaps.get(k))
udata.put("tjj_source_year",2023)
}
if(mzbMaps.get(k)){
udata.put("mzb_province_code", mzbMaps.get(k))
udata.put("mzb_source_year",2023)
}
lastRets << udata
}
}
saveToDB(lastRets, "省份")
}
def createCity(){
def cat = "城市"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def sameKeys = tjjKeys.intersect(mzbKeys)
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
sameKeys.each {k->
def v = tjjMaps.get(k)
lastRets << getAdministrativeCodeInfo(k, v, mzbMaps)
}
if(diffKeys){
diffKeys.each {k->
def v = tjjMaps.get(k)
lastRets <<getAdministrativeCodeInfo(k, v, mzbMaps)
}
}
//添加别名
lastRets.each {o->
def city = RegionData.formatCity(o.city)
if(city){
def cityObj = CommonDatautils.findAlCityDetail(city)
o.put("alias", cityObj.alias)
}else{
println o.province+" "+o.city +" 房价网无此城市"
}
}
//添加城市围栏
def cityPolylineMap = cityPolyline()
//添加pid
def provPidMap = createPidData("省份")
lastRets.each{o->
def shapeData = cityPolylineMap.get(o.city)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.province)
if(provId){
o.pid = provId
}
}
saveToDB(lastRets, "城市")
}
def createRegion(){
def cat = "区域"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def sameKeys = tjjKeys.intersect(mzbKeys)
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
sameKeys.each {k->
def v = tjjMaps.get(k)
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
// ud.source = "tjj|mzb"
lastRets << ud
}
if(diffKeys){
diffKeys.each {k->
def v = tjjMaps.get(k)
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
// if(v){
// ud.source = "tjj"
// }else{
// ud.source = "mzb"
// }
lastRets << ud
}
}
//添加别名
def regionMap = CommonDatautils.getAllCityRegions()
lastRets.each {o->
def city = RegionData.formatCity(o.city)
def district = RegionData.formatRegion(city, o.district)
if(city && district){
def cMap = regionMap.get(city)
if(cMap){
def regions = cMap.get(district)
o.put("alias", regions)
}
}else{
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
}
}
//添加pid
def provPidMap = createPidData("城市")
//添加区域围栏
def regionPolylineMap = regionPolyline()
lastRets.each{o->
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.city)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, "区域")
}
def createTown(){
def cat = "乡镇街道"
def tjjMaps = findAllProcincesFromTJJ(cat)
def lastRets = []
tjjMaps.each {k,v->
def ud = getAdministrativeCodeInfo(k, v, null)
lastRets << ud
}
//添加pid
def provPidMap = createPidData("区域")
//添加区域围栏
def regionPolylineMap = townPolyline()
lastRets.each{o->
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.city)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, "区域")
}
def getAdministrativeCodeInfo(key, val, mzbMaps){
def udata = new BasicDBObject()
def keys = key.split("\\|@\\|")
def vals = null
if(val){
vals = val.split("\\|@\\|")
}
def mzb_vals = null
if(mzbMaps && mzbMaps.containsKey(key)){
def mzb_val_str = mzbMaps.get(key)
mzb_vals = mzb_val_str.split("\\|@\\|")
}
if(keys.size()>=1){
udata.put("province", keys[0])
if(vals && vals.size()>=1){
udata.put("tjj_province_code", vals[0])
udata.put("tjj_source_year",2023)
}
if(mzb_vals && mzb_vals.size()>=1){
udata.put("mzb_province_code", mzb_vals[0])
udata.put("mzb_source_year",2023)
}
}
if(keys.size()>=2){
udata.put("city", keys[1])
if(vals && vals.size()>=2){
udata.put("tjj_city_code", vals[1])
}
if(mzb_vals && mzb_vals.size()>=2){
udata.put("mzb_city_code", mzb_vals[1])
}
}
if(keys.size()>=3){
udata.put("district", keys[2])
if(vals && vals.size()>=3) {
udata.put("tjj_district_code", vals[2])
}
if(mzb_vals && mzb_vals.size()>=3){
udata.put("mzb_district_code", mzb_vals[2])
}
}
return udata
}
def saveToDB(lastRets, type){
def dataList = []
for(def ret : lastRets){
ret.put("c_date", new Date())
ret.put("m_date", new Date())
ret.put("type", type)
dataList << ret
if (dataList.size() >= 5000) {
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
dataList.clear();
}
}
if (dataList.size() > 0) {
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
dataList.clear();
}
}
def createPidData(def type){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type))
def map = [:]
def pids = []
def step = 2
if(type == "城市"){
step = 4
}else if(type == "区域"){
step = 6
}
for(def d : dbcursor){
def pid = null
def code = null
if(d.mzb_province_code){
if(type == "省份") {
code = d.mzb_province_code.toString()
}else if(type == "城市"){
code = d.mzb_city_code.toString()
}else if(type == "区域"){
code = d.mzb_district_code.toString()
}
}else if(d.tjj_province_code){
if(type == "省份") {
code = d.tjj_province_code.toString()
}else if(type == "城市"){
code = d.tjj_city_code.toString()
}else if(ype == "区域"){
code = d.tjj_district_code.toString()
}
}
pid = code.substring(0,step)
if(pids.contains(d.city+"-"+pid)){
pid = code
}
if(!pids.contains(d.city+"-"+pid)){
pids << d.city+"-"+pid
}
if(type == "省份"){
map.put(d.province, pid)
}else if(type == "城市"){
map.put(d.city, pid)
}else if(type == "区域"){
map.put(d.city+"@"+d.district, pid)
}
}
return map;
}
def findAllProcincesFromTJJ(cat){
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
def match = null
if(cat=="省"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code'))
}else if(cat=="城市"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code'))
}else if(cat=="区域"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
}else if(cat=="乡镇街道"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("name", '$name').append("code", '$code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
// AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(match, group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city_name").toString()!="省直辖县级行政区划"){
def key = obj.get("province_name")
def value = obj.get("province_code")
if(cat=="城市"){
key = key +"|@|"+ obj.get("city_name")
value = value +"|@|"+ obj.get("city_code")
}else if(cat=="区域"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
}else if(cat=="乡镇街道"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("code")
}
int count = (Integer) result.get("count");
map.put(key, value)
}
}
return map;
}
def findAllProcincesFromMZB(cat){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
if(cat == "省"){
match = new BasicDBObject('$match', new BasicDBObject("level", "省份"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code'))
}else if(cat == "城市"){
match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code'))
}else if(cat == "区域"){
match = new BasicDBObject('$match', new BasicDBObject("level", "区域"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city").toString()!="省直辖县级行政区划"){
def key = obj.get("province")
def value = obj.get("mca_province_code")
if(cat == "城市"){
key = key +"|@|"+obj.get("city")
value = value + "|@|" + obj.get("mca_city_code")
}else if(cat == "区域"){
key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district")
value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code")
}
int count = (Integer) result.get("count");
map.put(key, value)
}
}
return map;
}
def cityPolyline(){
def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject())
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.name+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
map.put(d.name, udata)
}
return map;
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
def townPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
}
@@ -135,6 +135,26 @@ class FangjiaClientUtils {
return jsonObj; return jsonObj;
} }
public static def format(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){ public static def arroundPrice(city,address){
def jsonObj = null def jsonObj = null
def content = null def content = null
@@ -228,9 +248,9 @@ class FangjiaClientUtils {
def url = null; def url = null;
try { try {
address = address.replace("{","").replace("\\","") address = address.replace("{","").replace("\\","")
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" // url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" // url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){ if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8") url = url + "&city="+URLEncoder.encode(city,"UTF-8")
} }
@@ -33,12 +33,12 @@ class AddPriceTest {
task.run(); task.run();
} }
def run(){ def run(){
def filePath = "C:\\Users\\gaojuan\\Desktop\\7Ô³ɽ»°¸Àýbadcase.xlsx" def filePath = "D:\\数据\\建行\\房产估值\\估家待优化小区 - 副本.xlsx"
// addGXDPrice(filePath) // addGXDPrice(filePath)
// addCcbPrice(filePath) addCcbPrice(filePath)
// addThirdPrice(filePath) // addThirdPrice(filePath)
// addFjwPrice(filePath) // addFjwPrice(filePath)
exportPkPrice() // exportPkPrice()
// addRegion() // addRegion()
@@ -153,15 +153,15 @@ class AddPriceTest {
} }
def addCcbPrice(filepath){ def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d-> ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['M']){ if(d['A']){
def cityId = cityCodeDataMap.get(d['N']) // def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['M'], cityId) def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(map){ if(map){
d['AK'] = map.avg_price // d['AK'] = map.avg_price
d['AL'] = map.calc_type d['Q'] = map.calc_type
d['AM'] = map.credit_value // d['AM'] = map.credit_value
} }
} }
}) })
@@ -227,12 +227,12 @@ class AddPriceTest {
def addGXDPrice(filepath){ def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d -> ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) { if (!d['K']) {
def dist = CommonDatautils.findDistrictById(d['E']) def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){ if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['L'] = gxdPrice d['K'] = gxdPrice
} }
} }
}) })
@@ -256,14 +256,14 @@ class AddPriceTest {
def addFjwPrice(path){ def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d-> ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){ if(!d['P']){
def dist = CommonDatautils.findDistrictById(d['E']) def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){ if(dist){
// d['N'] = dist.region // d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd")) def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){ if(dbo){
// d['K'] = dbo.avg_price d['P'] = dbo.avg_price
d['K'] = dbo.avg_price_2b // d['J'] = dbo.avg_price_2b
} }
} }
} }
@@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject import net.sf.json.JSONObject
import okhttp3.*; import okhttp3.*;
import java.io.IOException; import java.io.IOException;
@@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder; import okhttp3.Request.Builder;
class QiliangTest { class QiliangTest {
String proxyHost = 'tunnel2.qg.net';
Integer proxyPort = 14186;
String proxyUsername = '113BBC06';
String proxyPassword = '3B2FBD05B0AF';
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
// static main(args) { static main(args) {
// def task = new QiliangTest() QiliangTest task = new QiliangTest();
//// task.run() // task.run()
// task.getHttpResult("扬州扬子江北路399号11041")
// }
def run(){//地址测试-2024.3.4-结果.xlsx def data = [
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> 'address': '浙江省湖州市德清县莫干山开发区中小工业园区',
if(!row['N']){ 'type': '2'
def address = null ]
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ def s = task.getProxyResult(data);
address = row['B']+row['C'] print(s);
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
} }
def getHttpResult(address){
def data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{ String getProxyResult(data){
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") // 将 JSON 对象转换为字符串
def response = JSONObject.fromObject(res) String jsonData = new JsonOutput().toJson(data);
println response // 创建代理对象
return response Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
}catch(Exception e){
e.printStackTrace() // 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
} }
return null; return null;
} }
final static String proxyIp = "tunnel2.qg.net";
final static Integer proxyPort = 14186;
final static String authKey = "113BBC06";
final static String password = "3B2FBD05B0AF";
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
// public String request(params) throws IOException {
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
// OkHttpClient client = new OkHttpClient.Builder()
// .proxy(proxy)
// .proxyAuthenticator((route, response){
// String credential = Credentials.basic(authKey, password);
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
// }).
// build();
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
//
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
//
// Response response = client.newCall(request).execute();
// return response.body().string();
//
//// return client.newCall(request).execute();
// }
public static void main(String[] args) {
def proxyHost = "tunnel2.qg.net"
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
// 创建代理服务器的连接
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
// 打开代理连接
def proxyConnection = proxy.openConnection()
// 创建目标URL的连接
def url = new URL(targetUrl)
def connection = url.openConnection(proxy)
// 设置请求方法
connection.setRequestMethod("GET")
// 添加请求头(可选)
connection.setRequestProperty("Content-Type", "application/json")
// 构建查询参数
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
// 将查询参数附加到目标URL
def finalUrl = "${targetUrl}?${queryString}"
// 发送请求
connection.connect()
// 获取响应
def responseCode = connection.getResponseCode()
def responseMessage = connection.getResponseMessage()
// 读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def responseContent = reader.readLine()
reader.close()
println "Response Code: ${responseCode}"
println "Response Message: ${responseMessage}"
println "Response Content: ${responseContent}"
}
} }
@@ -19,17 +19,17 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
//def s = getTownshipInfo("118.702988,39.740979") //def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx" def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
ExcelUtils.modify(filePath, { row -> ExcelUtils.modify(filePath, { row ->
// if(!row['I']){ if(!row['I'] && row['I']=='宝应县'){
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
// if(rets){ if(rets){
// def obj = findOneByName(rets, row['C']+"人民政府") def obj = findOneByName(rets, row['C']+"人民政府")
// if(obj){ if(obj){
// row['I'] = obj.lng row['I'] = obj.lng
// row['J'] = obj.lat row['J'] = obj.lat
// println row['C'] + " "+obj println row['C'] + " "+obj
// } }
// } }
// } }
if(row['D'] && !row['F']){ if(row['D'] && !row['F']){
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL) def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
@@ -0,0 +1,181 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest1 {
def proxyHost = 'tunnel2.qg.net'
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
static main(args) {
def task = new QiliangTest()
// task.run()
def data = [
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
'type': '1'
]
def s = task.getProxyResult(data)
print(s)
}
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
}
JSONObject getHttpResult(address){
JSONObject data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
JSONObject response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
}
return null;
}
def getProxyResult(data){
// 将 JSON 对象转换为字符串
def jsonData = new JsonOutput().toJson(data)
// 创建代理对象
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
// 创建 URL 对象
def url = new URL(targetUrl)
// 创建连接对象
def connection = url.openConnection(proxy)
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST")
// 允许输出流
connection.setDoOutput(true)
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json")
// 发送请求参数
def outputStream = connection.getOutputStream()
outputStream.write(jsonData.getBytes("UTF-8"))
outputStream.flush()
outputStream.close()
// 读取响应
def responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def response = new StringBuilder()
String line
while ((line = reader.readLine()) != null) {
response.append(line)
}
reader.close()
return response.toString()
} else {
println "HTTP request failed with response code ${responseCode}"
}
return null
}
}
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data import com.fangjia.script.xingyebank.Data
@@ -34,16 +35,18 @@ class PoiWash {
public static def regions = [] public static def regions = []
public static def nameKeys = null public static def nameKeys = null
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) { static main(args) {
def task = new PoiWash() def task = new PoiWash()
// task.updateId() // task.updateId()
// task.run() // task.run()
// task.synch() // task.synch()
// task.check() // task.check()
task.deduplicate() // task.deduplicate()
// task.updateEsMapPoi() // task.updateEsMapPoi()
// task.checkSpecialSymbols() // task.checkSpecialSymbols()
// task.splitRoad() task.splitRoad()
} }
def updateId(){ def updateId(){
@@ -452,9 +455,10 @@ class PoiWash {
def splitRoad(){ def splitRoad(){
//"ÈýÃ÷ÊÐ", //"ÈýÃ÷ÊÐ",
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"] def citys = CommonDatautils.findAllCity()
for(def city : citys) { for(def city : citys) {
def datas = findPoiByQuery(city, ['city','address']) def cityObj = fullCityMap.get(city)
def datas = findPoiByQuery(cityObj.city, ['city','address'])
splitRoadRun(datas) splitRoadRun(datas)
println city +" Íê³É" println city +" Íê³É"
} }
@@ -472,18 +476,15 @@ class PoiWash {
void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address) def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
if(jsonObject && jsonObject.containsKey("result")){ def street = jsonObject.get("street")
def dataMap = jsonObject.get("result") def streetNumber = jsonObject.get("streetNumber")
if(dataMap.get("street")){ if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
updateTotalCount.addAndGet(1) updateTotalCount.addAndGet(1)
def udata = new BasicDBObject() def udata = new BasicDBObject()
udata.put("street", dataMap.get("street")) udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
if(dataMap.get("streetNumber")){ udata.put("m_date", new Date())
udata.put("street_number", dataMap.get("streetNumber")) MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
} }
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" " println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
} catch (Exception e) { } catch (Exception e) {
@@ -26,8 +26,8 @@ class SeaweedDataSynch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL) DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020) public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
@@ -40,7 +40,7 @@ class SeaweedDataSynch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
@@ -58,10 +58,11 @@ class SeaweedDataSynch extends BaseBatch{
public void process(Object args) { public void process(Object args) {
run() // run()
// println "全国district, shop, office同步完成" // println "全国district, shop, office同步完成"
//民政部编码上传 //民政部编码上传
//codeRun() codeRun()
//println "民政部省,市,区同步完成" //println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate() //println new ObjectId("622c433afb459ab08fdab460").getDate()
@@ -346,24 +347,32 @@ class SeaweedDataSynch extends BaseBatch{
def codeRun(){ def codeRun(){
// def q = new BasicDBObject()
// def dbcursor = mca_administrative_division_coll.find(q)
// def rets = []
// for(def it : dbcursor){
// def udata = new BasicDBObject()
// udata.put("_id", it._id)
// udata.put("province", it.province)
// udata.put("city", it.city)
// udata.put("district", it.district)
// udata.put("mca_province_code", it.province_code)
// udata.put("mca_city_code", it.city_code)
// udata.put("mca_district_code", it.district_code)
// udata.put("level", it.level)
// udata.put("c_date", it.c_date)
// rets << udata
//// administrative_division_100_CONN.save(udata)
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
// }
// administrative_division_100_CONN.insert(rets)
//
def q = new BasicDBObject() def q = new BasicDBObject()
def dbcursor = mca_administrative_division_coll.find(q) q.put("province","°ÄÃÅÌØ±ðÐÐÕþÇø")
def rets = [] def dbcursor = administrative_division_2022_CONN.find(q)
for(def it : dbcursor){ for(def d : dbcursor){
def udata = new BasicDBObject() administrative_division_100_CONN.save(d)
udata.put("_id", it._id)
udata.put("province", it.province)
udata.put("city", it.city)
udata.put("district", it.district)
udata.put("mca_province_code", it.province_code)
udata.put("mca_city_code", it.city_code)
udata.put("mca_district_code", it.district_code)
udata.put("level", it.level)
udata.put("c_date", it.c_date)
rets << udata
// administrative_division_100_CONN.save(udata)
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
} }
administrative_division_100_CONN.insert(rets)
} }
} }
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
* 8:華硊摯華硊梗靡脤梑 * 8:華硊摯華硊梗靡脤梑
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑 * 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
* 10:⑽秞⑤饜 * 10:⑽秞⑤饜
*11:open½Ó¿ÚÆ¥Åä * 11:open½Ó¿ÚÆ¥Åä
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶 * 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
*/ */
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)