diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index a7b589cf9..cb4abf8cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -75,18 +75,22 @@ for(def d : datas){ map.area = d['F'].toString().toDouble() map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) map.avg_price = (int)(d['H'].toString().toDouble()) - map.sell_price = (int)(d['I'].toString().toDouble()*10000) - def trade_date = d['J'] - if(trade_date){ - map.trade_date = trade_date - }else{ - def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30") - map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") + if(d['I']){ + map.sell_price = (int)(d['I'].toString().toDouble()*10000) } +// def trade_date = d['J'] +// if(trade_date){ +// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd') +// }else{ + def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") +// } + map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() + println map DEAL_PRICE_COLL.save(map) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 4d473e36a..05a5447e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -69,51 +69,51 @@ class BigdataDealBatch extends BaseBatch{ addDealPriceMatch(city); } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - for(def city : citys){ - index ++ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) - println "历史id加载量:"+dealIds.size() - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成 " + rows.size() - -// delPgData() - saveToPgSql(rows) - -// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") -// saveToPgSql2(datas) +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// for(def city : citys){ +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) +// println "历史id加载量:"+dealIds.size() +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +//// delPgData() +// saveToPgSql(rows) +// +//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +//// saveToPgSql2(datas) } def readTxt(def fileName){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index fca28ce3e..023cc157d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -388,19 +388,21 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" - def citys = ["469005"]//CcbCommonUtils.citys_all +// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all + def citys = CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() - for(def cityid : citys) { + for(def city : citys) { + def cityid = cityCodeDatas.get(city) def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 - println cityid +" "+rets.size() + println city +" "+rets.size() if(rets){ - set.add(cityid) + set.add(city) } for(def m : rets){ -// def cityId = cityCodeDatas.get(city) + // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // sqlInstance236.executeUpdate(sql) def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" @@ -409,7 +411,7 @@ class DealPriceDataProcess { total = r.count }) if(total>0){ - println cityid +" "+m.commid+" (按价格)删除:"+total + println city +" "+m.commid+" (按价格)删除:"+total def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" pgsqlInstance.executeUpdate(sql) }else{ @@ -419,7 +421,7 @@ class DealPriceDataProcess { total = r.count }) if(total>0){ - println cityId +" "+m.commid+" (按小区)删除:"+total + println city +" "+m.commid+" (按小区)删除:"+total def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " pgsqlInstance.executeUpdate(sql) } @@ -450,7 +452,7 @@ class DealPriceDataProcess { } def findPeGteThirdParty2(cityid, yearmonth){ - def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20" + def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25" def rets = [] pgsqlInstance.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 2adcb8b71..99bed78ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-10") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d00970d5a..371e260ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,70 +1289,66 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx" -// -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ -//// def region = RegionData.formatRegion(d['B'], d['C']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") -//// if(dist){ -//// d['G'] = dist.region -//// d['H'] = dist._id.toString() -//// } -//// } -// if(!d['I']){ -// def dist = CommonDatautils.findDistrictById(d['C']) +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls" + +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['C']){ +// def region = RegionData.formatRegion(d['B'], d['C']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // if(dist){ -// d['F'] = dist.region -//// d['H'] = dist.name -// d['G'] = dist.status -//// if(d['A']=="天门" || d['A']=="迪庆"){ -//// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200) -//// d['K'] = val.toInteger() -//// } +// d['G'] = dist.region +// d['H'] = dist._id.toString() // } // } -// } -//},0) - -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - if(d['B'] && !d['D']){ - def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district") - if(jsonObject != null && jsonObject.credit>0) { - d['D'] = jsonObject.get("city") - d['E'] = jsonObject.get("district") - d['F'] = jsonObject.get("name") - def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) - if(dbo){ - d['G'] = dbo.status - d['H'] = dbo.name_cat - d['I'] = dbo.estate_type2 - d['J'] = "接口匹配" + if(!d['L']){ + def dist = CommonDatautils.findDistrictById(d['C']) + if(dist){ + d['L'] = dist.region + d['M'] = dist.name + d['N'] = dist.status } } + } +},0) -// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district") -// if(!objs){ -// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district") -// } -// if(objs && objs.size()==1){ -// def o = objs[0] -// if(o){ -// d['D'] = o.city -// d['E'] = o.region -// d['F'] = o.name -// d['G'] = o.status -// d['H'] = o.name_cat -// d['I'] = o.estate_type2 -// d['J'] = "别名" +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// if(d['B'] && !d['D']){ +// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district") +// if(jsonObject != null && jsonObject.credit>0) { +// d['D'] = jsonObject.get("city") +// d['E'] = jsonObject.get("district") +// d['F'] = jsonObject.get("name") +// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) +// if(dbo){ +// d['G'] = dbo.status +// d['H'] = dbo.name_cat +// d['I'] = dbo.estate_type2 +// d['J'] = "接口匹配" // } // } - - - } -}) +// +//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district") +//// if(!objs){ +//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district") +//// } +//// if(objs && objs.size()==1){ +//// def o = objs[0] +//// if(o){ +//// d['D'] = o.city +//// d['E'] = o.region +//// d['F'] = o.name +//// d['G'] = o.status +//// d['H'] = o.name_cat +//// d['I'] = o.estate_type2 +//// d['J'] = "别名" +//// } +//// } +// +// +// } +//}) //def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx" //ExcelUtils.multiThreadingModify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index d2ad41623..286172afd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" } @@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" } @@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" } @@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" } @@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" } @@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" } @@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { rets << ret } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e0e58814f..233a8e0f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData +import com.fangjia.datastore.driver.FacetResult import com.fangjia.httpclient.UserAgents import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.looplineinfo.InsidePolygon @@ -8,11 +9,16 @@ import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.JacksonUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.AggregationOutput +import com.mongodb.client.MongoCursor +import com.mongodb.client.model.Accumulators +import com.mongodb.client.model.Aggregates import groovy.json.JsonSlurper import net.sf.json.JSONArray import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils +import org.bson.Document import org.bson.types.ObjectId import com.fangjia.core.util.HttpRequest; @@ -110,6 +116,7 @@ class CommonDatautils { FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) } + public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009") public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -1459,6 +1466,28 @@ class CommonDatautils { return obj; } + static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + + BasicDBObject query = new BasicDBObject(); + query.putAt(QueryOperators.IN, [name]); + q.put("mapping_alias", query); + + def dbcursor = SEAWEED_COLL.find(q); + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + static def findSeaweeds(city, cat, name_cat){ DBObject q = new BasicDBObject(); q.put('city',city) @@ -4601,4 +4630,6 @@ class CommonDatautils { + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictMatch.groovy new file mode 100644 index 000000000..ee24ae8ae --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictMatch.groovy @@ -0,0 +1,3 @@ +package com.fangjia.script.fuzhou + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 893ad9876..585e7bc0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -29,11 +29,11 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() -// task.priceMarkupRun(); + task.priceMarkupRun(); // task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() - task.testNewPrice() +// task.testNewPrice() } def margeExcel(){ @@ -58,7 +58,7 @@ class FzDataResultStat { } def priceMarkupRun(){ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15"); - def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) // def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) @@ -82,7 +82,7 @@ class FzDataResultStat { } // List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) - def list2 = findFzDistrictStat(lastMonth) + def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"))) def map2 = [:] for(def obj : list2) { def key = obj.city + "##" + obj.region + "##" + obj.name diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index ad2cf8430..f96bf753a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index a2ec7c80b..90d781a2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject @@ -17,7 +18,60 @@ class DevProcess { static main(args) { def task = new DevProcess(); - task.run() +// task.run() + task.parseCityAndDist() + } + + def parseCityAndDist(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d-> + if(!d['J']){ +// def obj = getMapInfo(null, d['A']+d['C']) +// if(obj){ +// d['E'] = obj.city.toString() +// d['F'] = obj.district.toString() +// d['G'] = obj.formatted_address.toString() +// d['H'] = obj.province.toString() +// } + + def obj = FangjiaClientUtils.formatNinelevel(null, d['C']); + if(obj) { + def jsonObject = obj.get("result") + + if (jsonObject) { + if (jsonObject.get("province") && jsonObject.get("province") != "null") { + d['J'] = jsonObject.get("province") + } + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + d['K'] = jsonObject.get("city") + } + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + d['L'] = jsonObject.get("district") + } + } + } + } + }) + } + + def getMapInfo(city, address){ + def json = GaoDeApiUtils.geo(city, address) + println json + if(json && json.status=="1"){ + def ret = json.geocodes + if(ret){ + def obj = ret.get(0) + def str = obj.location + def level = obj.level + +// if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){ + def arrs = str.split(",") + return ["lng":arrs[0], "lat":arrs[1],"province":obj.province,"city":obj.city, "district":obj.district, "formatted_address":obj.formatted_address] +// }else{ +// println address +" "+level +// return null +// } + } + } } def run(){ def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 2a638980a..60d12e13b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -32,7 +33,9 @@ class GXDdistrictMatch { // task.updateDistStatus()//恢复name_cat不正常数据 // task.saveToDB() //保存新增小区 - task.restoreDist()//恢复小区status=0 +// task.restoreDist()//恢复小区status=0 + + task.runMatch() } def addOldMatch(){ @@ -133,6 +136,22 @@ class GXDdistrictMatch { }) } + def runMatch(){ + def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx + def city = "福州" + ExcelUtils.modify(fileName, { d -> + println d['C'] + if(d['B'] && d['A']){ + def region = RegionData.formatRegion(city, d['B']) + if (region) { + d = match(city, region, d) + } else { + println d['A'] + "#" + d['E'] + " 区域无法格式化" + } + } + }) + } + def updateDistStatus(){ def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx" def index = 0 @@ -187,27 +206,32 @@ class GXDdistrictMatch { } def saveToDB(){ - def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" +// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx" ExcelUtils.modify(fileName, { d -> - if(d['U'] && d['AG']!='过滤'){ + if(d['A']){ def udata = new BasicDBObject() - udata.put("city", d['AD']) - udata.put("region", d['AE']) - def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("city", d['A']) + udata.put("region", d['B']) + def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") udata.put("name", name) def alias = [] - if(name!=d['C']){ - alias << d['C'] - } - if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ - alias << d['AI'] + if(d['G'] && name!=d['G']){ + alias << d['G'] } +// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ +// alias << d['AI'] +// } if(alias){ udata.put("alias", alias) } - udata.put("address", d['AF']) + def addr = AddressUtil.formatAddress(d['A'], d['D']) + if(addr){ + udata.put("address", addr) + } + udata.put("status", "0") udata.put("name_cat", "0") udata.put("cat", "district") @@ -215,11 +239,11 @@ class GXDdistrictMatch { def estateType2 = "普通住宅" def type = "" - if(d['AH']){ - type = d['AH'] - }else{ - type = d['Y'] - } +// if(d['AH']){ +// type = d['AH'] +// }else{ +// type = d['Y'] +// } if(type){ if(type.contains("别墅")){ estateType2 = "别墅" @@ -228,12 +252,12 @@ class GXDdistrictMatch { } } udata.put("estateType2", estateType2) - if(d['AA']){ - Long lng = d['AA']*10000000000 + if(d['F']){ + Long lng = d['F']*10000000000 udata.put("lng", lng) } - if(d['AB']){ - Long lat = d['AB'] * 10000000000 + if(d['E']){ + Long lat = d['E'] * 10000000000 udata.put("lat", lat) } @@ -265,7 +289,7 @@ class GXDdistrictMatch { if(!d['U']){ d = matchApi(city, region, d, "property", false) if(!d['U']){ - d = matchApi(city, region, d, null, true) + d = matchApi(city, region, d, null, false) } } @@ -285,14 +309,18 @@ class GXDdistrictMatch { } def matchApi(city, region, d, cat, useMap){ - def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) - if(!jsonObject){ + def jsonObject = null + if(!jsonObject && d['H']){ + jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) + } + + if(!jsonObject && d['C']){ jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap) if(!jsonObject && d['D']){ jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap) } } - if(jsonObject != null && jsonObject.credit>0) { + if(jsonObject != null) { d['U'] = "接口匹配" d['V'] = jsonObject.get("city") d['W'] = jsonObject.get("district") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy new file mode 100644 index 000000000..4d865d728 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +class TempExportData { + + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new TempExportData() + task.run() + } + + def run() { + def citys = CommonDatautils.findAllCity() + def rets = [] + rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"] + for (def city : citys) { + def cityRets = getResults(city) + rets.addAll(cityRets) + } + ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据") + } + + def getResults(city) { + def rets = [] + def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + final AtomicInteger count_ = new AtomicInteger(0) + + for (def i = 0; i < dists.size(); i++) { + def d = dists[i] + fixedThreadPool.execute( + new Runnable() { + void run() { + count_.addAndGet(1) + try { + def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district") + if (objs) { + def ret = [] + ret << d._id.toString() + ret << d.city + ret << d.region + ret << d.name + ret << d.address + + def str = "" + for (def o : objs) { + str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n" + } + ret << str + ret << objs.size() + rets << ret + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + } + ); + } + latch.await() + println(city +" "+rets.size()) + return rets; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy new file mode 100644 index 000000000..c5debfdd2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -0,0 +1,722 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.AggregationOutput +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.AggregateIterable +import org.bson.Document + +/** + * 乡镇街道中间表 + * 包含市,区,乡镇街道别名,来源 统计局community表 + */ +class TownMiddleProcess { + + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) + DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL) + + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") + DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline") + DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + + def fullCityMap //= CommonDatautils.queryFullNameCity() + + static main(args) { + def task = new TownMiddleProcess(); + task.run() + } + def run(){ + //创建省份 +// createProvince() + //创建城市 +// createCity() + //创建区域 +// createRegion() + //创建乡镇街道 +// createTown() + //修改乡镇街道别名 +// updateTownAlias() + //中间表同步到100 +// synchTown() + //添加地级市标志 + addCityFlag() + } + + + def createProvince(){ + def cat = "省" + def tjjMaps = findAllProcincesFromTJJ(cat) + def mzbMaps = findAllProcincesFromMZB(cat) + + def tjjKeys = tjjMaps.keySet() + def mzbKeys = mzbMaps.keySet() + + def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys) + def lastRets = [] + tjjMaps.each {key, val-> + def udata = new BasicDBObject() + udata.put("province", key) + udata.put("tjj_province_code", val) + udata.put("mzb_province_code", mzbMaps.get(key)) + udata.put("tjj_source_year",2023) + udata.put("mzb_source_year",2023) + lastRets << udata + } + if(diffKeys){ + diffKeys.each {k-> + def udata = new BasicDBObject() + udata.put("province", k) + if(tjjMaps.get(k)){ + udata.put("tjj_province_code", tjjMaps.get(k)) + udata.put("tjj_source_year",2023) + } + if(mzbMaps.get(k)){ + udata.put("mzb_province_code", mzbMaps.get(k)) + udata.put("mzb_source_year",2023) + } + lastRets << udata + } + } + saveToDB(lastRets, "省份") + } + + + def createCity(){ + def cat = "城市" + def tjjMaps = findAllProcincesFromTJJ(cat) + def mzbMaps = findAllProcincesFromMZB(cat) + def tjjKeys = tjjMaps.keySet() + def mzbKeys = mzbMaps.keySet() + def sameKeys = tjjKeys.intersect(mzbKeys) + def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys) + def lastRets = [] + sameKeys.each {k-> + def v = tjjMaps.get(k) + lastRets << getAdministrativeCodeInfo(k, v, mzbMaps) + } + + if(diffKeys){ + diffKeys.each {k-> + def v = tjjMaps.get(k) + lastRets < + def city = RegionData.formatCity(o.city) + if(city){ + def cityObj = CommonDatautils.findAlCityDetail(city) + o.put("alias", cityObj.alias) + }else{ + println o.province+" "+o.city +" 房价网无此城市" + } + } + //添加城市围栏 + def cityPolylineMap = cityPolyline() + //添加pid + def provPidMap = createPidData("省份") + lastRets.each{o-> + def shapeData = cityPolylineMap.get(o.city) + if(shapeData){ + o.shape = shapeData.shape + o.location = shapeData.location + } + def provId = provPidMap.get(o.province) + if(provId){ + o.pid = provId + } + } + saveToDB(lastRets, cat) + } + + + + def createRegion(){ + def cat = "区域" + def tjjMaps = findAllProcincesFromTJJ(cat) + def mzbMaps = findAllProcincesFromMZB(cat) + def tjjKeys = tjjMaps.keySet() + def mzbKeys = mzbMaps.keySet() + def sameKeys = tjjKeys.intersect(mzbKeys) + def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys) + def lastRets = [] + sameKeys.each {k-> + def v = tjjMaps.get(k) + def ud = getAdministrativeCodeInfo(k, v, mzbMaps) +// ud.source = "tjj|mzb" + lastRets << ud + } + + if(diffKeys){ + diffKeys.each {k-> + def v = tjjMaps.get(k) + def ud = getAdministrativeCodeInfo(k, v, mzbMaps) +// if(v){ +// ud.source = "tjj" +// }else{ +// ud.source = "mzb" +// } + lastRets << ud + } + } + //添加别名 + def regionMap = CommonDatautils.getAllCityRegions() + lastRets.each {o-> + def city = RegionData.formatCity(o.city) + def district = RegionData.formatRegion(city, o.district) + if(city && district){ + def cMap = regionMap.get(city) + if(cMap){ + def regions = cMap.get(district) + o.put("alias", regions) + } + }else{ + println o.province+" "+o.city+" "+ o.district +" 房价网无此城市" + } + } + + //添加pid + def provPidMap = createPidData("城市") + + //添加区域围栏 + def regionPolylineMap = regionPolyline() + + lastRets.each{o-> + def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district) + if(shapeData){ + o.shape = shapeData.shape + o.location = shapeData.location + } + def provId = provPidMap.get(o.city) + if(provId){ + o.pid = provId + } + } + //是否为县级市 + saveToDB(lastRets, cat) + } + + def createTown(){ + def cat = "乡镇街道" + def tjjMaps = findAllProcincesFromTJJ(cat) + def lastRets = [] + def govMap = findAllGovTown() + tjjMaps.each {k,v-> + def ud = getAdministrativeCodeInfo(k, v, null) + if(govMap && govMap.containsKey(k)){ + ud.put("has_gov", govMap.get(k)) + } + lastRets << ud + } + + //添加区域围栏 + def regionPolylineMap = townPolyline() + //添加pid + def provPidMap = createPidData("区域") + + lastRets.each{o-> + def shapeData = getStreetPolyline(regionPolylineMap, o, o.district) + if(!shapeData){ + def newRegion = o.district + def standardRegion = RegionData.formatRegion(o.city,o.district) + if(standardRegion && standardRegion!=newRegion){ + newRegion = standardRegion + shapeData = getStreetPolyline(regionPolylineMap, o, newRegion) + } + } + if(shapeData){ + o.shape = shapeData.shape + if(shapeData.location){ + o.location = shapeData.location + } + } + def provId = provPidMap.get(o.city+"@"+o.district) + if(provId){ + o.pid = provId + } + } + //是否为县级市 + saveToDB(lastRets, cat) + } + + def getStreetPolyline(regionPolylineMap, o, district){ + def shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+o.street) + if(!shapeData){ + def streetKey = o.street + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"街道" + }else if(streetKey.toString().endsWith("乡")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇" + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"镇" + }else if(streetKey.toString().endsWith("地区")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("地区"))+"镇" + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + if(!shapeData){ + streetKey = o.street + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"乡" + }else if(streetKey.toString().endsWith("乡")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇" + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"社区" + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + } + if(!shapeData){ + streetKey = o.street + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"地区" + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"乡" + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + } + if(!shapeData){ + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇")) + }else if(streetKey.toString().endsWith("乡")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡")) + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道")) + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + } + } + return shapeData + } + + + def getAdministrativeCodeInfo(key, val, mzbMaps){ + def udata = new BasicDBObject() + def keys = key.split("\\|@\\|") + def vals = null + if(val){ + vals = val.split("\\|@\\|") + } + + def mzb_vals = null + if(mzbMaps && mzbMaps.containsKey(key)){ + def mzb_val_str = mzbMaps.get(key) + mzb_vals = mzb_val_str.split("\\|@\\|") + } + if(keys.size()>=1){ + udata.put("province", keys[0]) + if(vals && vals.size()>=1){ + udata.put("tjj_province_code", vals[0]) + udata.put("tjj_source_year",2023) + } + if(mzb_vals && mzb_vals.size()>=1){ + udata.put("mzb_province_code", mzb_vals[0]) + udata.put("mzb_source_year",2023) + } + } + + + if(keys.size()>=2){ + udata.put("city", keys[1]) + if(vals && vals.size()>=2){ + udata.put("tjj_city_code", vals[1]) + } + if(mzb_vals && mzb_vals.size()>=2){ + udata.put("mzb_city_code", mzb_vals[1]) + } + } + + + if(keys.size()>=3){ + udata.put("district", keys[2]) + if(vals && vals.size()>=3) { + udata.put("tjj_district_code", vals[2]) + } + if(mzb_vals && mzb_vals.size()>=3){ + udata.put("mzb_district_code", mzb_vals[2]) + } + } + + if(keys.size()>=4){ + udata.put("street", keys[3]) + if(vals && vals.size()>=4) { + udata.put("tjj_street_code", vals[3]) + } + if(mzb_vals && mzb_vals.size()>=4){ + udata.put("mzb_street_code", mzb_vals[3]) + } + } + + return udata + } + + + def updateTownAlias(){ + ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇别名问题审核交付_0418.xlsx",{ row-> + if(row['G'] && row['G']=="不处理"){ + def q = new BasicDBObject() + q.put("city",row['C']) + q.put("district",row['D']) + q.put("street",row['E']) + def d = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(d){ + def alias = [] + alias << row['F'] + def udata = new BasicDBObject() + udata.put("alias", alias) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + }else{ + println row['C']+" "+row['D']+" "+row["E"] + } + } + }) + } + + + def synchTown(){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000) + def datas = [] + for(def d : dbcursor){ + datas << d + if (datas.size() >= 2000) { + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + println datas.size() + datas.clear(); + } + } + + if (datas.size() >= 0) { + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + datas.clear(); + } + } + + + def saveToDB(lastRets, type){ + def dataList = [] + for(def ret : lastRets){ + ret.put("c_date", new Date()) + ret.put("m_date", new Date()) + ret.put("type", type) + dataList << ret + + if (dataList.size() >= 5000) { + ADMINISTRATIVE_MAPPING_COLL.insert(dataList); + dataList.clear(); + } + } + + if (dataList.size() > 0) { + ADMINISTRATIVE_MAPPING_COLL.insert(dataList); + dataList.clear(); + } + } + + + def createPidData(def type){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type)).batchSize(5000) + def map = [:] + def pids = [] + def step = 2 + if(type == "城市"){ + step = 4 + }else if(type == "区域"){ + step = 6 + } + for(def d : dbcursor){ + def pid = null + def code = null + if(d.mzb_province_code){ + if(type == "省份") { + code = d.mzb_province_code.toString() + }else if(type == "城市"){ + code = d.mzb_city_code.toString() + }else if(type == "区域"){ + code = d.mzb_district_code.toString() + } + }else if(d.tjj_province_code){ + if(type == "省份") { + code = d.tjj_province_code.toString() + }else if(type == "城市"){ + code = d.tjj_city_code.toString() + }else if(type == "区域"){ + code = d.tjj_district_code.toString() + } + } + + pid = code.substring(0,step) + if(pids.contains(d.city+"-"+pid)){ + pid = code + } + if(!pids.contains(d.city+"-"+pid)){ + pids << d.city+"-"+pid + } + if(type == "省份"){ + map.put(d.province, pid) + }else if(type == "城市"){ + map.put(d.city, pid) + }else if(type == "区域"){ + map.put(d.city+"@"+d.district, pid) + } + } + return map; + } + + def findAllGovTown(){ + def q = new BasicDBObject(); + q.put("has_gov", new BasicDBObject('$ne', '')) + def keys = new BasicDBObject() + keys.put("province_name",1) + keys.put("city_name",1) + keys.put("area_name", 1) + keys.put("street_name", 1) + keys.put("has_gov", 1) + def dbcursor = COMMUNITY_CONN.find(q,keys).batchSize(5000) + def map = [:] + for(def d : dbcursor){ + def key = d.province_name+"|@|"+d.city_name+"|@|"+d.area_name+"|@|"+d.street_name + if(!map.containsKey(key)){ + map.put(key, d.has_gov) + } + } + return map; + } + + + def findAllProcincesFromTJJ(cat){ + // 定义聚合管道,按 city 字段分组,并计算每个城市的数量 + DBObject groupFields = new BasicDBObject(); + def match = null + if(cat=="省"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code')) + }else if(cat=="城市"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code')) + }else if(cat=="区域"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code')) + }else if(cat=="乡镇街道"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("street_name", '$street_name').append("street_code", '$street_code')) + } + groupFields.put("count", new BasicDBObject('$sum', 1)); + DBObject group = new BasicDBObject('$group', groupFields); + + // 执行聚合查询 + AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group)); + + def map = [:] + // 遍历结果 + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + if(obj.get("city_name").toString()!="省直辖县级行政区划"){ + def key = obj.get("province_name") + def value = obj.get("province_code") + if(cat=="城市"){ + key = key +"|@|"+ obj.get("city_name") + value = value +"|@|"+ obj.get("city_code") + }else if(cat=="区域"){ + key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name") + value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code") + }else if(cat=="乡镇街道"){ + key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("street_name") + value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("street_code") + } + int count = (Integer) result.get("count"); + map.put(key, value) + } + } + return map; + } + + + def findAllProcincesFromMZB(cat){ + + BasicDBObject match = null; + // 定义聚合管道,按 city 字段分组,并计算每个城市的数量 + DBObject groupFields = new BasicDBObject(); + + if(cat == "省"){ + match = new BasicDBObject('$match', new BasicDBObject("level", "省份")); + groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code')) + }else if(cat == "城市"){ + match = new BasicDBObject('$match', new BasicDBObject("level", "城市")); + groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code')) + }else if(cat == "区域"){ + match = new BasicDBObject('$match', new BasicDBObject("level", "区域")); + groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code')) + } + + + groupFields.put("count", new BasicDBObject('$sum', 1)); + DBObject group = new BasicDBObject('$group', groupFields); + + // 执行聚合查询 + AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group)); + def map = [:] + // 遍历结果 + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + if(obj.get("city").toString()!="省直辖县级行政区划"){ + def key = obj.get("province") + def value = obj.get("mca_province_code") + if(cat == "城市"){ + key = key +"|@|"+obj.get("city") + value = value + "|@|" + obj.get("mca_city_code") + }else if(cat == "区域"){ + key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district") + value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code") + } + int count = (Integer) result.get("count"); + map.put(key, value) + } + } + return map; + } + + def cityPolyline(){ + def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject()) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.name+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + map.put(d.name, udata) + } + return map; + } + + def regionPolyline(){ + def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district")) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + + def fullCity = fullCityMap.get(d.city) + map.put(fullCity.city+"|@|"+d.region, udata) + } + return map; + } + + def townPolyline(){ + def dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject()).batchSize(5000) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+d.town+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + map.put(d.city+"|@|"+d.fjw_region+"|@|"+d.town, udata) + } + return map; + } + + def addCityFlag(){ +// ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row-> +// if(row['A']){ +// def q = new BasicDBObject("type", "城市") +// q.put("city", row['A']) +// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) +// if(obj){ +// def udata = new BasicDBObject() +// udata.put("prefecture_level_city", "是") +// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) +// }else{ +// println row['A']+" 没找到该城市" +// } +// } +// }); + + ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row-> + if(row['A']){ + def q = new BasicDBObject("type", "区域") + q.put("district", row['A']) + def count = ADMINISTRATIVE_MAPPING_COLL.count(q) + if(count && count == 1){ + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + def udata = new BasicDBObject() + udata.put("county_level_city", "是") + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + }else{ + println row['A']+" 没找到该城市" + } + } + },1); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 260ef5cb7..a664aefa9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -135,6 +135,26 @@ class FangjiaClientUtils { return jsonObj; } + public static def format(def city, def address){ + def jsonObj = null + def content = null + try { + if(address){ + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static def arroundPrice(city,address){ def jsonObj = null def content = null @@ -228,9 +248,9 @@ class FangjiaClientUtils { def url = null; try { address = address.replace("{","").replace("\\","") - url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" // url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" -// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" + url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index db5b55b54..7b7b54d0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,12 +33,12 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\新疆3月待优化小区 - 副本.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) -// addFjwPrice(filePath) - exportPkPrice() + addFjwPrice(filePath) +// exportPkPrice() // addRegion() @@ -153,15 +153,15 @@ class AddPriceTest { } def addCcbPrice(filepath){ - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(filepath,{ d-> - if(d['M']){ - def cityId = cityCodeDataMap.get(d['N']) - def map = findAvgPrice(d['M'], cityId) + if(d['A']){ +// def cityId = cityCodeDataMap.get(d['N']) + def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger()) if(map){ - d['AK'] = map.avg_price - d['AL'] = map.calc_type - d['AM'] = map.credit_value +// d['AK'] = map.avg_price + d['P'] = map.calc_type +// d['AM'] = map.credit_value } } }) @@ -227,12 +227,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['B']) { - def dist = CommonDatautils.findDistrictById(d['E']) + if (!d['K']) { + def dist = CommonDatautils.findDistrictById(d['C']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['L'] = gxdPrice + d['K'] = gxdPrice } } }) @@ -256,14 +256,14 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['E']) + if(!d['P']){ + def dist = CommonDatautils.findDistrictById(d['G']) if(dist){ // d['N'] = dist.region - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd")) + def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ -// d['K'] = dbo.avg_price - d['K'] = dbo.avg_price_2b + d['O'] = dbo.avg_price +// d['J'] = dbo.avg_price_2b } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index dade0fa1f..9801c3588 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.HttpClient +import groovy.json.JsonOutput import net.sf.json.JSONObject import okhttp3.*; import java.io.IOException; @@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit; import okhttp3.Request.Builder; class QiliangTest { + String proxyHost = 'tunnel2.qg.net'; + Integer proxyPort = 14186; + String proxyUsername = '113BBC06'; + String proxyPassword = '3B2FBD05B0AF'; + String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; -// static main(args) { -// def task = new QiliangTest() -//// task.run() -// task.getHttpResult("扬州扬子江北路399号11041") -// } + static main(args) { + QiliangTest task = new QiliangTest(); +// task.run() - def run(){//地址测试-2024.3.4-结果.xlsx - ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> - if(!row['N']){ - def address = null - if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ - address = row['B']+row['C'] - }else{ - address = row['C'] - } - if(address){ - def result = getHttpResult(address) - if(result && result.code==20000){ - def data = result.data - if(data){ - if(data.province && data.province!='null'){ - row['N'] = data.province - } - if(data.city && data.city!='null'){ - row['O'] = data.city - } - if(data.district && data.district!='null'){ - row['P'] = data.district - } - if(data.town && data.town!='null'){ - row['Q'] = data.town - } - if(data.village && data.village!='null'){ - row['R'] = data.village - } - - if(data.areaCode && data.areaCode!='null'){ - row['S'] = data.areaCode - } - - if(data.farmer == "0"){ - row['T'] = "非农" - }else{ - row['T'] = "农" - } - if(data.isGovt == "0"){ - row['U'] = "否" - }else{ - row['U'] = "是" - } - - def list = data.indusParkList - if(list && list!="null"){ - def s = "" - for(def r : list){ - s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" - } - row['V'] = s - } - if(data.govtAddr && data.govtAddr!='null'){ - row['W'] = data.govtAddr?data.govtAddr:'' - } - - } - } - } - } - }) + def data = [ + 'address': '浙江省湖州市德清县莫干山开发区中小工业园区', + 'type': '2' + ] + def s = task.getProxyResult(data); + print(s); } - def getHttpResult(address){ - def data = new JSONObject() - data.put("address", address) - data.put("type","2") - Map headers = new HashMap(); - headers.put("Accept", "application/json, text/plain, */*"); - headers.put("Accept-Encoding", "gzip, deflate"); - headers.put("Accept-Language","zh-CN,zh;q=0.9") - headers.put("Content-Type","application/json;charset=UTF-8") - headers.put("Host","146.56.217.44:9010") - headers.put("Origin","http://146.56.217.44:8088") - headers.put("Proxy-Connection","keep-alive") - headers.put("Referer","http://146.56.217.44:8088/") - headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") - HttpRequest request1 = new HttpRequest() - try{ - String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") - def response = JSONObject.fromObject(res) - println response - return response - }catch(Exception e){ - e.printStackTrace() + + String getProxyResult(data){ + // 将 JSON 对象转换为字符串 + String jsonData = new JsonOutput().toJson(data); + // 创建代理对象 + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + + // 创建 URL 对象 + URL url = new URL(targetUrl); + + // 创建连接对象 + URLConnection connection = url.openConnection(proxy); + + // 设置代理认证 + java.net.Authenticator.setDefault(new java.net.Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()) + } + }) + + // 设置请求方法为 POST + connection.setRequestMethod("POST"); + + // 允许输出流 + connection.setDoOutput(true); + + // 设置请求头 + connection.setRequestProperty("Content-Type", "application/json"); + + // 发送请求参数 + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(jsonData.getBytes("UTF-8")); + outputStream.flush(); + outputStream.close(); + + // 读取响应 + int responseCode = connection.getResponseCode() + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + return response.toString(); + } else { + println "HTTP request failed with response code ${responseCode}"; } return null; } - - final static String proxyIp = "tunnel2.qg.net"; - final static Integer proxyPort = 14186; - final static String authKey = "113BBC06"; - final static String password = "3B2FBD05B0AF"; - public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8"); - -// public String request(params) throws IOException { -// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort)); -// OkHttpClient client = new OkHttpClient.Builder() -// .proxy(proxy) -// .proxyAuthenticator((route, response){ -// String credential = Credentials.basic(authKey, password); -// return response.request().newBuilder().header("Proxy-Authorization", credential).build(); -// }). -// build(); -//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build(); -// -// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params)); -// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build(); -// -// Response response = client.newCall(request).execute(); -// return response.body().string(); -// -//// return client.newCall(request).execute(); -// } - public static void main(String[] args) { - def proxyHost = "tunnel2.qg.net" - def proxyPort = 14186 - def proxyUsername = '113BBC06' - def proxyPassword = '3B2FBD05B0AF' - - def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign' - def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2'] - - -// 创建代理服务器的连接 - def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}") - -// 打开代理连接 - def proxyConnection = proxy.openConnection() - -// 创建目标URL的连接 - def url = new URL(targetUrl) - def connection = url.openConnection(proxy) - -// 设置请求方法 - connection.setRequestMethod("GET") - -// 添加请求头(可选) - connection.setRequestProperty("Content-Type", "application/json") - -// 构建查询参数 - def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&') - -// 将查询参数附加到目标URL - def finalUrl = "${targetUrl}?${queryString}" - -// 发送请求 - connection.connect() - -// 获取响应 - def responseCode = connection.getResponseCode() - def responseMessage = connection.getResponseMessage() - -// 读取响应内容 - def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) - def responseContent = reader.readLine() - reader.close() - - println "Response Code: ${responseCode}" - println "Response Message: ${responseMessage}" - println "Response Content: ${responseContent}" - } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy index b0258b431..980d2863d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy @@ -19,17 +19,17 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun //def s = getTownshipInfo("118.702988,39.740979") def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx" ExcelUtils.modify(filePath, { row -> -// if(!row['I']){ -// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") -// if(rets){ -// def obj = findOneByName(rets, row['C']+"人民政府") -// if(obj){ -// row['I'] = obj.lng -// row['J'] = obj.lat -// println row['C'] + " "+obj -// } -// } -// } + if(!row['I'] && row['I']=='宝应县'){ + def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") + if(rets){ + def obj = findOneByName(rets, row['C']+"人民政府") + if(obj){ + row['I'] = obj.lng + row['J'] = obj.lat + println row['C'] + " "+obj + } + } + } if(row['D'] && !row['F']){ def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/test.groovy new file mode 100644 index 000000000..b186739ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/test.groovy @@ -0,0 +1,181 @@ +package com.fangjia.script.gj.village + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.HttpClient +import groovy.json.JsonOutput +import net.sf.json.JSONObject +import okhttp3.*; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.concurrent.TimeUnit; + + +import okhttp3.Request.Builder; + +class QiliangTest1 { + def proxyHost = 'tunnel2.qg.net' + def proxyPort = 14186 + def proxyUsername = '113BBC06' + def proxyPassword = '3B2FBD05B0AF' + String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; + + static main(args) { + def task = new QiliangTest() +// task.run() + + def data = [ + 'address': '江苏省扬州市宝应县安宜镇宝应大道88号', + 'type': '1' + ] + def s = task.getProxyResult(data) + print(s) + } + + def run(){//地址测试-2024.3.4-结果.xlsx + ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> + if(!row['N']){ + def address = null + if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ + address = row['B']+row['C'] + }else{ + address = row['C'] + } + if(address){ + def result = getHttpResult(address) + if(result && result.code==20000){ + def data = result.data + if(data){ + if(data.province && data.province!='null'){ + row['N'] = data.province + } + if(data.city && data.city!='null'){ + row['O'] = data.city + } + if(data.district && data.district!='null'){ + row['P'] = data.district + } + if(data.town && data.town!='null'){ + row['Q'] = data.town + } + if(data.village && data.village!='null'){ + row['R'] = data.village + } + + if(data.areaCode && data.areaCode!='null'){ + row['S'] = data.areaCode + } + + if(data.farmer == "0"){ + row['T'] = "非农" + }else{ + row['T'] = "农" + } + if(data.isGovt == "0"){ + row['U'] = "否" + }else{ + row['U'] = "是" + } + + def list = data.indusParkList + if(list && list!="null"){ + def s = "" + for(def r : list){ + s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" + } + row['V'] = s + } + if(data.govtAddr && data.govtAddr!='null'){ + row['W'] = data.govtAddr?data.govtAddr:'' + } + + } + } + } + } + }) + } + + JSONObject getHttpResult(address){ + JSONObject data = new JSONObject() + data.put("address", address) + data.put("type","2") + + Map headers = new HashMap(); + headers.put("Accept", "application/json, text/plain, */*"); + headers.put("Accept-Encoding", "gzip, deflate"); + headers.put("Accept-Language","zh-CN,zh;q=0.9") + headers.put("Content-Type","application/json;charset=UTF-8") + headers.put("Host","146.56.217.44:9010") + headers.put("Origin","http://146.56.217.44:8088") + headers.put("Proxy-Connection","keep-alive") + headers.put("Referer","http://146.56.217.44:8088/") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + + HttpRequest request1 = new HttpRequest() + try{ + String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") + JSONObject response = JSONObject.fromObject(res) + println response + return response + }catch(Exception e){ + e.printStackTrace() + } + return null; + } + + + def getProxyResult(data){ + // 将 JSON 对象转换为字符串 + def jsonData = new JsonOutput().toJson(data) + // 创建代理对象 + def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)) + + // 创建 URL 对象 + def url = new URL(targetUrl) + + // 创建连接对象 + def connection = url.openConnection(proxy) + + // 设置代理认证 + java.net.Authenticator.setDefault(new java.net.Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()) + } + }) + + // 设置请求方法为 POST + connection.setRequestMethod("POST") + + // 允许输出流 + connection.setDoOutput(true) + + // 设置请求头 + connection.setRequestProperty("Content-Type", "application/json") + + // 发送请求参数 + def outputStream = connection.getOutputStream() + outputStream.write(jsonData.getBytes("UTF-8")) + outputStream.flush() + outputStream.close() + + // 读取响应 + def responseCode = connection.getResponseCode() + if (responseCode == HttpURLConnection.HTTP_OK) { + def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) + def response = new StringBuilder() + String line + while ((line = reader.readLine()) != null) { + response.append(line) + } + reader.close() + return response.toString() + } else { + println "HTTP request failed with response code ${responseCode}" + } + return null + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy index 302cd02e2..45e3614bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.xingyebank.Data @@ -34,16 +35,18 @@ class PoiWash { public static def regions = [] public static def nameKeys = null + def fullCityMap = CommonDatautils.queryFullNameCity() + static main(args) { def task = new PoiWash() // task.updateId() // task.run() // task.synch() // task.check() - task.deduplicate() +// task.deduplicate() // task.updateEsMapPoi() // task.checkSpecialSymbols() -// task.splitRoad() + task.splitRoad() } def updateId(){ @@ -452,9 +455,10 @@ class PoiWash { def splitRoad(){ //"三明市", - def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"] + def citys = CommonDatautils.findAllCity() for(def city : citys) { - def datas = findPoiByQuery(city, ['city','address']) + def cityObj = fullCityMap.get(city) + def datas = findPoiByQuery(cityObj.city, ['city','address']) splitRoadRun(datas) println city +" 完成" } @@ -472,18 +476,15 @@ class PoiWash { void run() { try{ totalCount.addAndGet(1) - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address) - if(jsonObject && jsonObject.containsKey("result")){ - def dataMap = jsonObject.get("result") - if(dataMap.get("street")){ - updateTotalCount.addAndGet(1) - def udata = new BasicDBObject() - udata.put("street", dataMap.get("street")) - if(dataMap.get("streetNumber")){ - udata.put("street_number", dataMap.get("streetNumber")) - } - MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata)) - } + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){ + updateTotalCount.addAndGet(1) + def udata = new BasicDBObject() + udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber")) + udata.put("m_date", new Date()) + MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata)) } println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" " } catch (Exception e) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 0d1e2f3b3..6f704a4a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -26,8 +26,8 @@ class SeaweedDataSynch extends BaseBatch{ def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) - DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) - + DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) + DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL) public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020) @@ -40,7 +40,7 @@ class SeaweedDataSynch extends BaseBatch{ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") - public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) + public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020) DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) @@ -58,10 +58,11 @@ class SeaweedDataSynch extends BaseBatch{ public void process(Object args) { - run() +// run() // println "全国district, shop, office同步完成" //民政部编码上传 - //codeRun() + codeRun() + //println "民政部省,市,区同步完成" //println new ObjectId("622c433afb459ab08fdab460").getDate() @@ -346,24 +347,32 @@ class SeaweedDataSynch extends BaseBatch{ def codeRun(){ +// def q = new BasicDBObject() +// def dbcursor = mca_administrative_division_coll.find(q) +// def rets = [] +// for(def it : dbcursor){ +// def udata = new BasicDBObject() +// udata.put("_id", it._id) +// udata.put("province", it.province) +// udata.put("city", it.city) +// udata.put("district", it.district) +// udata.put("mca_province_code", it.province_code) +// udata.put("mca_city_code", it.city_code) +// udata.put("mca_district_code", it.district_code) +// udata.put("level", it.level) +// udata.put("c_date", it.c_date) +// rets << udata +//// administrative_division_100_CONN.save(udata) +//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false) +// } +// administrative_division_100_CONN.insert(rets) +// def q = new BasicDBObject() - def dbcursor = mca_administrative_division_coll.find(q) - def rets = [] - for(def it : dbcursor){ - def udata = new BasicDBObject() - udata.put("_id", it._id) - udata.put("province", it.province) - udata.put("city", it.city) - udata.put("district", it.district) - udata.put("mca_province_code", it.province_code) - udata.put("mca_city_code", it.city_code) - udata.put("mca_district_code", it.district_code) - udata.put("level", it.level) - udata.put("c_date", it.c_date) - rets << udata -// administrative_division_100_CONN.save(udata) -// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false) + q.put("province","澳门特别行政区") + def dbcursor = administrative_division_2022_CONN.find(q) + for(def d : dbcursor){ + administrative_division_100_CONN.save(d) + } - administrative_division_100_CONN.insert(rets) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 18fdb2d2f..0e5692cb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -46,7 +46,7 @@ import java.util.regex.Pattern * 8:地址及地址别名查找 * 9:地址及地址别名(不加区域限制)查找 * 10:拼音匹配 - *11:open接口匹配 + * 11:open接口匹配 * 14:拼音匹配,不加地址限制 */ @BatchOwner(Watcher.GaoJuan)