Merge branch 'script_gj' into 'master'

update

See merge request !833
This commit is contained in:
2024-04-30 15:41:42 +08:00
20 changed files with 1178 additions and 381 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -75,18 +75,22 @@ for(def d : datas){
map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J']
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
if(d['I']){
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
}
// def trade_date = d['J']
// if(trade_date){
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
// }else{
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
// }
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
println map
DEAL_PRICE_COLL.save(map)
}
@@ -69,51 +69,51 @@ class BigdataDealBatch extends BaseBatch{
addDealPriceMatch(city);
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
// delPgData()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
// println "历史id加载量:"+dealIds.size()
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
//// delPgData()
// saveToPgSql(rows)
//
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
//// saveToPgSql2(datas)
}
def readTxt(def fileName){
@@ -388,19 +388,21 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
def citys = ["469005"]//CcbCommonUtils.citys_all
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def cityid : citys) {
for(def city : citys) {
def cityid = cityCodeDatas.get(city)
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
println cityid +" "+rets.size()
println city +" "+rets.size()
if(rets){
set.add(cityid)
set.add(city)
}
for(def m : rets){
// def cityId = cityCodeDatas.get(city)
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
@@ -409,7 +411,7 @@ class DealPriceDataProcess {
total = r.count
})
if(total>0){
println cityid +" "+m.commid+" (按价格)删除:"+total
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
@@ -419,7 +421,7 @@ class DealPriceDataProcess {
total = r.count
})
if(total>0){
println cityId +" "+m.commid+" (按小区)删除:"+total
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
@@ -450,7 +452,7 @@ class DealPriceDataProcess {
}
def findPeGteThirdParty2(cityid, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20"
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-10")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1289,70 +1289,66 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
//// def region = RegionData.formatRegion(d['B'], d['C'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//// if(dist){
//// d['G'] = dist.region
//// d['H'] = dist._id.toString()
//// }
//// }
// if(!d['I']){
// def dist = CommonDatautils.findDistrictById(d['C'])
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['C']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
// if(dist){
// d['F'] = dist.region
//// d['H'] = dist.name
// d['G'] = dist.status
//// if(d['A']=="天门" || d['A']=="迪庆"){
//// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
//// d['K'] = val.toInteger()
//// }
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// }
// }
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['B'] && !d['D']){
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
if(jsonObject != null && jsonObject.credit>0) {
d['D'] = jsonObject.get("city")
d['E'] = jsonObject.get("district")
d['F'] = jsonObject.get("name")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['G'] = dbo.status
d['H'] = dbo.name_cat
d['I'] = dbo.estate_type2
d['J'] = "接口匹配"
if(!d['L']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
d['L'] = dist.region
d['M'] = dist.name
d['N'] = dist.status
}
}
}
},0)
// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
// if(!objs){
// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
// }
// if(objs && objs.size()==1){
// def o = objs[0]
// if(o){
// d['D'] = o.city
// d['E'] = o.region
// d['F'] = o.name
// d['G'] = o.status
// d['H'] = o.name_cat
// d['I'] = o.estate_type2
// d['J'] = "别名"
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
// if(jsonObject != null && jsonObject.credit>0) {
// d['D'] = jsonObject.get("city")
// d['E'] = jsonObject.get("district")
// d['F'] = jsonObject.get("name")
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
// if(dbo){
// d['G'] = dbo.status
// d['H'] = dbo.name_cat
// d['I'] = dbo.estate_type2
// d['J'] = "接口匹配"
// }
// }
}
})
//
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
//// if(!objs){
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
//// }
//// if(objs && objs.size()==1){
//// def o = objs[0]
//// if(o){
//// d['D'] = o.city
//// d['E'] = o.region
//// d['F'] = o.name
//// d['G'] = o.status
//// d['H'] = o.name_cat
//// d['I'] = o.estate_type2
//// d['J'] = "别名"
//// }
//// }
//
//
// }
//})
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
@@ -1,6 +1,7 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
@@ -8,11 +9,16 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest;
@@ -110,6 +116,7 @@ class CommonDatautils {
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -1459,6 +1466,28 @@ class CommonDatautils {
return obj;
}
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('status',0)
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, [name]);
q.put("mapping_alias", query);
def dbcursor = SEAWEED_COLL.find(q);
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
static def findSeaweeds(city, cat, name_cat){
DBObject q = new BasicDBObject();
q.put('city',city)
@@ -4601,4 +4630,6 @@ class CommonDatautils {
}
@@ -0,0 +1,3 @@
package com.fangjia.script.fuzhou
@@ -29,11 +29,11 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
// task.priceMarkupRun();
task.priceMarkupRun();
// task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
task.testNewPrice()
// task.testNewPrice()
}
def margeExcel(){
@@ -58,7 +58,7 @@ class FzDataResultStat {
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
@@ -82,7 +82,7 @@ class FzDataResultStat {
}
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
def map2 = [:]
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -32,7 +33,9 @@ class GXDdistrictMatch {
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
task.restoreDist()//恢复小区status=0
// task.restoreDist()//恢复小区status=0
task.runMatch()
}
def addOldMatch(){
@@ -133,6 +136,22 @@ class GXDdistrictMatch {
})
}
def runMatch(){
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
def city = "福州"
ExcelUtils.modify(fileName, { d ->
println d['C']
if(d['B'] && d['A']){
def region = RegionData.formatRegion(city, d['B'])
if (region) {
d = match(city, region, d)
} else {
println d['A'] + "#" + d['E'] + " 区域无法格式化"
}
}
})
}
def updateDistStatus(){
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
def index = 0
@@ -187,27 +206,32 @@ class GXDdistrictMatch {
}
def saveToDB(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['U'] && d['AG']!='过滤'){
if(d['A']){
def udata = new BasicDBObject()
udata.put("city", d['AD'])
udata.put("region", d['AE'])
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("city", d['A'])
udata.put("region", d['B'])
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(name!=d['C']){
alias << d['C']
}
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
alias << d['AI']
if(d['G'] && name!=d['G']){
alias << d['G']
}
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){
udata.put("alias", alias)
}
udata.put("address", d['AF'])
def addr = AddressUtil.formatAddress(d['A'], d['D'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
@@ -215,11 +239,11 @@ class GXDdistrictMatch {
def estateType2 = "普通住宅"
def type = ""
if(d['AH']){
type = d['AH']
}else{
type = d['Y']
}
// if(d['AH']){
// type = d['AH']
// }else{
// type = d['Y']
// }
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
@@ -228,12 +252,12 @@ class GXDdistrictMatch {
}
}
udata.put("estateType2", estateType2)
if(d['AA']){
Long lng = d['AA']*10000000000
if(d['F']){
Long lng = d['F']*10000000000
udata.put("lng", lng)
}
if(d['AB']){
Long lat = d['AB'] * 10000000000
if(d['E']){
Long lat = d['E'] * 10000000000
udata.put("lat", lat)
}
@@ -265,7 +289,7 @@ class GXDdistrictMatch {
if(!d['U']){
d = matchApi(city, region, d, "property", false)
if(!d['U']){
d = matchApi(city, region, d, null, true)
d = matchApi(city, region, d, null, false)
}
}
@@ -285,14 +309,18 @@ class GXDdistrictMatch {
}
def matchApi(city, region, d, cat, useMap){
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
if(!jsonObject){
def jsonObject = null
if(!jsonObject && d['H']){
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
}
if(!jsonObject && d['C']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
if(!jsonObject && d['D']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
}
}
if(jsonObject != null && jsonObject.credit>0) {
if(jsonObject != null) {
d['U'] = "接口匹配"
d['V'] = jsonObject.get("city")
d['W'] = jsonObject.get("district")
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class TempExportData {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new TempExportData()
task.run()
}
def run() {
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
for (def city : citys) {
def cityRets = getResults(city)
rets.addAll(cityRets)
}
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
}
def getResults(city) {
def rets = []
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
final AtomicInteger count_ = new AtomicInteger(0)
for (def i = 0; i < dists.size(); i++) {
def d = dists[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try {
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
if (objs) {
def ret = []
ret << d._id.toString()
ret << d.city
ret << d.region
ret << d.name
ret << d.address
def str = ""
for (def o : objs) {
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
}
ret << str
ret << objs.size()
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
println(city +" "+rets.size())
return rets;
}
}
@@ -0,0 +1,544 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
*/
class TownMiddleProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new TownMiddleProcess();
task.run()
}
def run(){
//创建省份
// createProvince()
//创建城市
// createCity()
//创建区域
// createRegion()
//创建乡镇街道
createTown()
}
def createProvince(){
def cat = "省"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
tjjMaps.each {key, val->
def udata = new BasicDBObject()
udata.put("province", key)
udata.put("tjj_province_code", val)
udata.put("mzb_province_code", mzbMaps.get(key))
udata.put("tjj_source_year",2023)
udata.put("mzb_source_year",2023)
lastRets << udata
}
if(diffKeys){
diffKeys.each {k->
def udata = new BasicDBObject()
udata.put("province", k)
if(tjjMaps.get(k)){
udata.put("tjj_province_code", tjjMaps.get(k))
udata.put("tjj_source_year",2023)
}
if(mzbMaps.get(k)){
udata.put("mzb_province_code", mzbMaps.get(k))
udata.put("mzb_source_year",2023)
}
lastRets << udata
}
}
saveToDB(lastRets, "省份")
}
def createCity(){
def cat = "城市"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def sameKeys = tjjKeys.intersect(mzbKeys)
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
sameKeys.each {k->
def v = tjjMaps.get(k)
lastRets << getAdministrativeCodeInfo(k, v, mzbMaps)
}
if(diffKeys){
diffKeys.each {k->
def v = tjjMaps.get(k)
lastRets <<getAdministrativeCodeInfo(k, v, mzbMaps)
}
}
//添加别名
lastRets.each {o->
def city = RegionData.formatCity(o.city)
if(city){
def cityObj = CommonDatautils.findAlCityDetail(city)
o.put("alias", cityObj.alias)
}else{
println o.province+" "+o.city +" 房价网无此城市"
}
}
//添加城市围栏
def cityPolylineMap = cityPolyline()
//添加pid
def provPidMap = createPidData("省份")
lastRets.each{o->
def shapeData = cityPolylineMap.get(o.city)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.province)
if(provId){
o.pid = provId
}
}
saveToDB(lastRets, "城市")
}
def createRegion(){
def cat = "区域"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def sameKeys = tjjKeys.intersect(mzbKeys)
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
sameKeys.each {k->
def v = tjjMaps.get(k)
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
// ud.source = "tjj|mzb"
lastRets << ud
}
if(diffKeys){
diffKeys.each {k->
def v = tjjMaps.get(k)
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
// if(v){
// ud.source = "tjj"
// }else{
// ud.source = "mzb"
// }
lastRets << ud
}
}
//添加别名
def regionMap = CommonDatautils.getAllCityRegions()
lastRets.each {o->
def city = RegionData.formatCity(o.city)
def district = RegionData.formatRegion(city, o.district)
if(city && district){
def cMap = regionMap.get(city)
if(cMap){
def regions = cMap.get(district)
o.put("alias", regions)
}
}else{
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
}
}
//添加pid
def provPidMap = createPidData("城市")
//添加区域围栏
def regionPolylineMap = regionPolyline()
lastRets.each{o->
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.city)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, "区域")
}
def createTown(){
def cat = "乡镇街道"
def tjjMaps = findAllProcincesFromTJJ(cat)
def lastRets = []
tjjMaps.each {k,v->
def ud = getAdministrativeCodeInfo(k, v, null)
lastRets << ud
}
//添加pid
def provPidMap = createPidData("区域")
//添加区域围栏
def regionPolylineMap = townPolyline()
lastRets.each{o->
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.city)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, "区域")
}
def getAdministrativeCodeInfo(key, val, mzbMaps){
def udata = new BasicDBObject()
def keys = key.split("\\|@\\|")
def vals = null
if(val){
vals = val.split("\\|@\\|")
}
def mzb_vals = null
if(mzbMaps && mzbMaps.containsKey(key)){
def mzb_val_str = mzbMaps.get(key)
mzb_vals = mzb_val_str.split("\\|@\\|")
}
if(keys.size()>=1){
udata.put("province", keys[0])
if(vals && vals.size()>=1){
udata.put("tjj_province_code", vals[0])
udata.put("tjj_source_year",2023)
}
if(mzb_vals && mzb_vals.size()>=1){
udata.put("mzb_province_code", mzb_vals[0])
udata.put("mzb_source_year",2023)
}
}
if(keys.size()>=2){
udata.put("city", keys[1])
if(vals && vals.size()>=2){
udata.put("tjj_city_code", vals[1])
}
if(mzb_vals && mzb_vals.size()>=2){
udata.put("mzb_city_code", mzb_vals[1])
}
}
if(keys.size()>=3){
udata.put("district", keys[2])
if(vals && vals.size()>=3) {
udata.put("tjj_district_code", vals[2])
}
if(mzb_vals && mzb_vals.size()>=3){
udata.put("mzb_district_code", mzb_vals[2])
}
}
return udata
}
def saveToDB(lastRets, type){
def dataList = []
for(def ret : lastRets){
ret.put("c_date", new Date())
ret.put("m_date", new Date())
ret.put("type", type)
dataList << ret
if (dataList.size() >= 5000) {
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
dataList.clear();
}
}
if (dataList.size() > 0) {
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
dataList.clear();
}
}
def createPidData(def type){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type))
def map = [:]
def pids = []
def step = 2
if(type == "城市"){
step = 4
}else if(type == "区域"){
step = 6
}
for(def d : dbcursor){
def pid = null
def code = null
if(d.mzb_province_code){
if(type == "省份") {
code = d.mzb_province_code.toString()
}else if(type == "城市"){
code = d.mzb_city_code.toString()
}else if(type == "区域"){
code = d.mzb_district_code.toString()
}
}else if(d.tjj_province_code){
if(type == "省份") {
code = d.tjj_province_code.toString()
}else if(type == "城市"){
code = d.tjj_city_code.toString()
}else if(ype == "区域"){
code = d.tjj_district_code.toString()
}
}
pid = code.substring(0,step)
if(pids.contains(d.city+"-"+pid)){
pid = code
}
if(!pids.contains(d.city+"-"+pid)){
pids << d.city+"-"+pid
}
if(type == "省份"){
map.put(d.province, pid)
}else if(type == "城市"){
map.put(d.city, pid)
}else if(type == "区域"){
map.put(d.city+"@"+d.district, pid)
}
}
return map;
}
def findAllProcincesFromTJJ(cat){
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
def match = null
if(cat=="省"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code'))
}else if(cat=="城市"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code'))
}else if(cat=="区域"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
}else if(cat=="乡镇街道"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("name", '$name').append("code", '$code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
// AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(match, group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city_name").toString()!="省直辖县级行政区划"){
def key = obj.get("province_name")
def value = obj.get("province_code")
if(cat=="城市"){
key = key +"|@|"+ obj.get("city_name")
value = value +"|@|"+ obj.get("city_code")
}else if(cat=="区域"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
}else if(cat=="乡镇街道"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("code")
}
int count = (Integer) result.get("count");
map.put(key, value)
}
}
return map;
}
def findAllProcincesFromMZB(cat){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
if(cat == "省"){
match = new BasicDBObject('$match', new BasicDBObject("level", "省份"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code'))
}else if(cat == "城市"){
match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code'))
}else if(cat == "区域"){
match = new BasicDBObject('$match', new BasicDBObject("level", "区域"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city").toString()!="省直辖县级行政区划"){
def key = obj.get("province")
def value = obj.get("mca_province_code")
if(cat == "城市"){
key = key +"|@|"+obj.get("city")
value = value + "|@|" + obj.get("mca_city_code")
}else if(cat == "区域"){
key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district")
value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code")
}
int count = (Integer) result.get("count");
map.put(key, value)
}
}
return map;
}
def cityPolyline(){
def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject())
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.name+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
map.put(d.name, udata)
}
return map;
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
def townPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
}
@@ -135,6 +135,26 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def format(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -228,9 +248,9 @@ class FangjiaClientUtils {
def url = null;
try {
address = address.replace("{","").replace("\\","")
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -33,12 +33,12 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\估家待优化小区 - 副本.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
exportPkPrice()
// exportPkPrice()
// addRegion()
@@ -153,15 +153,15 @@ class AddPriceTest {
}
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['M']){
def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['M'], cityId)
if(d['A']){
// def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(map){
d['AK'] = map.avg_price
d['AL'] = map.calc_type
d['AM'] = map.credit_value
// d['AK'] = map.avg_price
d['Q'] = map.calc_type
// d['AM'] = map.credit_value
}
}
})
@@ -227,12 +227,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictById(d['E'])
if (!d['K']) {
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['L'] = gxdPrice
d['K'] = gxdPrice
}
}
})
@@ -256,14 +256,14 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(!d['P']){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
// d['K'] = dbo.avg_price
d['K'] = dbo.avg_price_2b
d['P'] = dbo.avg_price
// d['J'] = dbo.avg_price_2b
}
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
@@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest {
String proxyHost = 'tunnel2.qg.net';
Integer proxyPort = 14186;
String proxyUsername = '113BBC06';
String proxyPassword = '3B2FBD05B0AF';
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
// static main(args) {
// def task = new QiliangTest()
//// task.run()
// task.getHttpResult("扬州扬子江北路399号11041")
// }
static main(args) {
QiliangTest task = new QiliangTest();
// task.run()
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
def data = [
'address': '浙江省湖州市德清县莫干山开发区中小工业园区',
'type': '2'
]
def s = task.getProxyResult(data);
print(s);
}
def getHttpResult(address){
def data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
def response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
String getProxyResult(data){
// 将 JSON 对象转换为字符串
String jsonData = new JsonOutput().toJson(data);
// 创建代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
}
return null;
}
final static String proxyIp = "tunnel2.qg.net";
final static Integer proxyPort = 14186;
final static String authKey = "113BBC06";
final static String password = "3B2FBD05B0AF";
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
// public String request(params) throws IOException {
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
// OkHttpClient client = new OkHttpClient.Builder()
// .proxy(proxy)
// .proxyAuthenticator((route, response){
// String credential = Credentials.basic(authKey, password);
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
// }).
// build();
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
//
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
//
// Response response = client.newCall(request).execute();
// return response.body().string();
//
//// return client.newCall(request).execute();
// }
public static void main(String[] args) {
def proxyHost = "tunnel2.qg.net"
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
// 创建代理服务器的连接
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
// 打开代理连接
def proxyConnection = proxy.openConnection()
// 创建目标URL的连接
def url = new URL(targetUrl)
def connection = url.openConnection(proxy)
// 设置请求方法
connection.setRequestMethod("GET")
// 添加请求头(可选)
connection.setRequestProperty("Content-Type", "application/json")
// 构建查询参数
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
// 将查询参数附加到目标URL
def finalUrl = "${targetUrl}?${queryString}"
// 发送请求
connection.connect()
// 获取响应
def responseCode = connection.getResponseCode()
def responseMessage = connection.getResponseMessage()
// 读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def responseContent = reader.readLine()
reader.close()
println "Response Code: ${responseCode}"
println "Response Message: ${responseMessage}"
println "Response Content: ${responseContent}"
}
}
@@ -19,17 +19,17 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
ExcelUtils.modify(filePath, { row ->
// if(!row['I']){
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
// if(rets){
// def obj = findOneByName(rets, row['C']+"人民政府")
// if(obj){
// row['I'] = obj.lng
// row['J'] = obj.lat
// println row['C'] + " "+obj
// }
// }
// }
if(!row['I'] && row['I']=='宝应县'){
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
if(rets){
def obj = findOneByName(rets, row['C']+"人民政府")
if(obj){
row['I'] = obj.lng
row['J'] = obj.lat
println row['C'] + " "+obj
}
}
}
if(row['D'] && !row['F']){
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
@@ -0,0 +1,181 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest1 {
def proxyHost = 'tunnel2.qg.net'
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
static main(args) {
def task = new QiliangTest()
// task.run()
def data = [
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
'type': '1'
]
def s = task.getProxyResult(data)
print(s)
}
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
}
JSONObject getHttpResult(address){
JSONObject data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
JSONObject response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
}
return null;
}
def getProxyResult(data){
// 将 JSON 对象转换为字符串
def jsonData = new JsonOutput().toJson(data)
// 创建代理对象
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
// 创建 URL 对象
def url = new URL(targetUrl)
// 创建连接对象
def connection = url.openConnection(proxy)
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST")
// 允许输出流
connection.setDoOutput(true)
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json")
// 发送请求参数
def outputStream = connection.getOutputStream()
outputStream.write(jsonData.getBytes("UTF-8"))
outputStream.flush()
outputStream.close()
// 读取响应
def responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def response = new StringBuilder()
String line
while ((line = reader.readLine()) != null) {
response.append(line)
}
reader.close()
return response.toString()
} else {
println "HTTP request failed with response code ${responseCode}"
}
return null
}
}
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
@@ -34,16 +35,18 @@ class PoiWash {
public static def regions = []
public static def nameKeys = null
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new PoiWash()
// task.updateId()
// task.run()
// task.synch()
// task.check()
task.deduplicate()
// task.deduplicate()
// task.updateEsMapPoi()
// task.checkSpecialSymbols()
// task.splitRoad()
task.splitRoad()
}
def updateId(){
@@ -452,9 +455,10 @@ class PoiWash {
def splitRoad(){
//"三明市",
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
def citys = CommonDatautils.findAllCity()
for(def city : citys) {
def datas = findPoiByQuery(city, ['city','address'])
def cityObj = fullCityMap.get(city)
def datas = findPoiByQuery(cityObj.city, ['city','address'])
splitRoadRun(datas)
println city +" 完成"
}
@@ -472,18 +476,15 @@ class PoiWash {
void run() {
try{
totalCount.addAndGet(1)
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
if(jsonObject && jsonObject.containsKey("result")){
def dataMap = jsonObject.get("result")
if(dataMap.get("street")){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("street", dataMap.get("street"))
if(dataMap.get("streetNumber")){
udata.put("street_number", dataMap.get("streetNumber"))
}
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
udata.put("m_date", new Date())
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
} catch (Exception e) {
@@ -26,8 +26,8 @@ class SeaweedDataSynch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
@@ -40,7 +40,7 @@ class SeaweedDataSynch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
@@ -58,10 +58,11 @@ class SeaweedDataSynch extends BaseBatch{
public void process(Object args) {
run()
// run()
// println "全国district, shop, office同步完成"
//民政部编码上传
//codeRun()
codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
@@ -346,24 +347,32 @@ class SeaweedDataSynch extends BaseBatch{
def codeRun(){
// def q = new BasicDBObject()
// def dbcursor = mca_administrative_division_coll.find(q)
// def rets = []
// for(def it : dbcursor){
// def udata = new BasicDBObject()
// udata.put("_id", it._id)
// udata.put("province", it.province)
// udata.put("city", it.city)
// udata.put("district", it.district)
// udata.put("mca_province_code", it.province_code)
// udata.put("mca_city_code", it.city_code)
// udata.put("mca_district_code", it.district_code)
// udata.put("level", it.level)
// udata.put("c_date", it.c_date)
// rets << udata
//// administrative_division_100_CONN.save(udata)
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
// }
// administrative_division_100_CONN.insert(rets)
//
def q = new BasicDBObject()
def dbcursor = mca_administrative_division_coll.find(q)
def rets = []
for(def it : dbcursor){
def udata = new BasicDBObject()
udata.put("_id", it._id)
udata.put("province", it.province)
udata.put("city", it.city)
udata.put("district", it.district)
udata.put("mca_province_code", it.province_code)
udata.put("mca_city_code", it.city_code)
udata.put("mca_district_code", it.district_code)
udata.put("level", it.level)
udata.put("c_date", it.c_date)
rets << udata
// administrative_division_100_CONN.save(udata)
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
q.put("province","°ÄÃÅÌØ±ðÐÐÕþÇø")
def dbcursor = administrative_division_2022_CONN.find(q)
for(def d : dbcursor){
administrative_division_100_CONN.save(d)
}
administrative_division_100_CONN.insert(rets)
}
}
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
* 8:華硊摯華硊梗靡脤梑
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
* 10:⑽秞⑤饜
*11:open½Ó¿ÚÆ¥Åä
* 11:open½Ó¿ÚÆ¥Åä
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
*/
@BatchOwner(Watcher.GaoJuan)