update
This commit is contained in:
+12
-8
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -75,18 +75,22 @@ for(def d : datas){
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
if(d['I']){
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
}
|
||||
|
||||
// def trade_date = d['J']
|
||||
// if(trade_date){
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
// }else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
println map
|
||||
DEAL_PRICE_COLL.save(map)
|
||||
}
|
||||
|
||||
|
||||
+45
-45
@@ -69,51 +69,51 @@ class BigdataDealBatch extends BaseBatch{
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
// delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
// saveToPgSql2(datas)
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
// println "历史id加载量:"+dealIds.size()
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
//// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
//
|
||||
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
//// saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
|
||||
+10
-8
@@ -388,19 +388,21 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||
def citys = ["469005"]//CcbCommonUtils.citys_all
|
||||
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def cityid : citys) {
|
||||
for(def city : citys) {
|
||||
def cityid = cityCodeDatas.get(city)
|
||||
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||
println cityid +" "+rets.size()
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
set.add(cityid)
|
||||
set.add(city)
|
||||
}
|
||||
for(def m : rets){
|
||||
// def cityId = cityCodeDatas.get(city)
|
||||
|
||||
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
// sqlInstance236.executeUpdate(sql)
|
||||
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
@@ -409,7 +411,7 @@ class DealPriceDataProcess {
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println cityid +" "+m.commid+" (按价格)删除:"+total
|
||||
println city +" "+m.commid+" (按价格)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
@@ -419,7 +421,7 @@ class DealPriceDataProcess {
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println cityId +" "+m.commid+" (按小区)删除:"+total
|
||||
println city +" "+m.commid+" (按小区)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
@@ -450,7 +452,7 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def findPeGteThirdParty2(cityid, yearmonth){
|
||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20"
|
||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
|
||||
def rets = []
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-10")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+52
-56
@@ -1289,70 +1289,66 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
//// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//// if(dist){
|
||||
//// d['G'] = dist.region
|
||||
//// d['H'] = dist._id.toString()
|
||||
//// }
|
||||
//// }
|
||||
// if(!d['I']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['C']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
// if(dist){
|
||||
// d['F'] = dist.region
|
||||
//// d['H'] = dist.name
|
||||
// d['G'] = dist.status
|
||||
//// if(d['A']=="天门" || d['A']=="迪庆"){
|
||||
//// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
|
||||
//// d['K'] = val.toInteger()
|
||||
//// }
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['B'] && !d['D']){
|
||||
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
d['D'] = jsonObject.get("city")
|
||||
d['E'] = jsonObject.get("district")
|
||||
d['F'] = jsonObject.get("name")
|
||||
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
if(dbo){
|
||||
d['G'] = dbo.status
|
||||
d['H'] = dbo.name_cat
|
||||
d['I'] = dbo.estate_type2
|
||||
d['J'] = "接口匹配"
|
||||
if(!d['L']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(dist){
|
||||
d['L'] = dist.region
|
||||
d['M'] = dist.name
|
||||
d['N'] = dist.status
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
|
||||
// if(!objs){
|
||||
// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
|
||||
// }
|
||||
// if(objs && objs.size()==1){
|
||||
// def o = objs[0]
|
||||
// if(o){
|
||||
// d['D'] = o.city
|
||||
// d['E'] = o.region
|
||||
// d['F'] = o.name
|
||||
// d['G'] = o.status
|
||||
// d['H'] = o.name_cat
|
||||
// d['I'] = o.estate_type2
|
||||
// d['J'] = "别名"
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// if(d['B'] && !d['D']){
|
||||
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
|
||||
// if(jsonObject != null && jsonObject.credit>0) {
|
||||
// d['D'] = jsonObject.get("city")
|
||||
// d['E'] = jsonObject.get("district")
|
||||
// d['F'] = jsonObject.get("name")
|
||||
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
// if(dbo){
|
||||
// d['G'] = dbo.status
|
||||
// d['H'] = dbo.name_cat
|
||||
// d['I'] = dbo.estate_type2
|
||||
// d['J'] = "接口匹配"
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
//
|
||||
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
|
||||
//// if(!objs){
|
||||
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
|
||||
//// }
|
||||
//// if(objs && objs.size()==1){
|
||||
//// def o = objs[0]
|
||||
//// if(o){
|
||||
//// d['D'] = o.city
|
||||
//// d['E'] = o.region
|
||||
//// d['F'] = o.name
|
||||
//// d['G'] = o.status
|
||||
//// d['H'] = o.name_cat
|
||||
//// d['I'] = o.estate_type2
|
||||
//// d['J'] = "别名"
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
//})
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
|
||||
+31
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
@@ -8,11 +9,16 @@ import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
@@ -110,6 +116,7 @@ class CommonDatautils {
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -1459,6 +1466,28 @@ class CommonDatautils {
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
|
||||
BasicDBObject query = new BasicDBObject();
|
||||
query.putAt(QueryOperators.IN, [name]);
|
||||
q.put("mapping_alias", query);
|
||||
|
||||
def dbcursor = SEAWEED_COLL.find(q);
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findSeaweeds(city, cat, name_cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put('city',city)
|
||||
@@ -4601,4 +4630,6 @@ class CommonDatautils {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
|
||||
+4
-4
@@ -29,11 +29,11 @@ class FzDataResultStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
// task.priceMarkupRun();
|
||||
task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
task.testNewPrice()
|
||||
// task.testNewPrice()
|
||||
}
|
||||
|
||||
def margeExcel(){
|
||||
@@ -58,7 +58,7 @@ class FzDataResultStat {
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
@@ -82,7 +82,7 @@ class FzDataResultStat {
|
||||
}
|
||||
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(lastMonth)
|
||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
||||
def map2 = [:]
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+53
-25
@@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -32,7 +33,9 @@ class GXDdistrictMatch {
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
task.restoreDist()//恢复小区status=0
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
task.runMatch()
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
@@ -133,6 +136,22 @@ class GXDdistrictMatch {
|
||||
})
|
||||
}
|
||||
|
||||
def runMatch(){
|
||||
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
|
||||
def city = "福州"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
println d['C']
|
||||
if(d['B'] && d['A']){
|
||||
def region = RegionData.formatRegion(city, d['B'])
|
||||
if (region) {
|
||||
d = match(city, region, d)
|
||||
} else {
|
||||
println d['A'] + "#" + d['E'] + " 区域无法格式化"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDistStatus(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
|
||||
def index = 0
|
||||
@@ -187,27 +206,32 @@ class GXDdistrictMatch {
|
||||
}
|
||||
|
||||
def saveToDB(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['U'] && d['AG']!='过滤'){
|
||||
if(d['A']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['AD'])
|
||||
udata.put("region", d['AE'])
|
||||
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def alias = []
|
||||
if(name!=d['C']){
|
||||
alias << d['C']
|
||||
}
|
||||
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
alias << d['AI']
|
||||
if(d['G'] && name!=d['G']){
|
||||
alias << d['G']
|
||||
}
|
||||
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
// alias << d['AI']
|
||||
// }
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
|
||||
udata.put("address", d['AF'])
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("cat", "district")
|
||||
@@ -215,11 +239,11 @@ class GXDdistrictMatch {
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
if(d['AH']){
|
||||
type = d['AH']
|
||||
}else{
|
||||
type = d['Y']
|
||||
}
|
||||
// if(d['AH']){
|
||||
// type = d['AH']
|
||||
// }else{
|
||||
// type = d['Y']
|
||||
// }
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
@@ -228,12 +252,12 @@ class GXDdistrictMatch {
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
if(d['AA']){
|
||||
Long lng = d['AA']*10000000000
|
||||
if(d['F']){
|
||||
Long lng = d['F']*10000000000
|
||||
udata.put("lng", lng)
|
||||
}
|
||||
if(d['AB']){
|
||||
Long lat = d['AB'] * 10000000000
|
||||
if(d['E']){
|
||||
Long lat = d['E'] * 10000000000
|
||||
udata.put("lat", lat)
|
||||
}
|
||||
|
||||
@@ -265,7 +289,7 @@ class GXDdistrictMatch {
|
||||
if(!d['U']){
|
||||
d = matchApi(city, region, d, "property", false)
|
||||
if(!d['U']){
|
||||
d = matchApi(city, region, d, null, true)
|
||||
d = matchApi(city, region, d, null, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,14 +309,18 @@ class GXDdistrictMatch {
|
||||
}
|
||||
|
||||
def matchApi(city, region, d, cat, useMap){
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
||||
if(!jsonObject){
|
||||
def jsonObject = null
|
||||
if(!jsonObject && d['H']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
||||
}
|
||||
|
||||
if(!jsonObject && d['C']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
|
||||
if(!jsonObject && d['D']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
|
||||
}
|
||||
}
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
if(jsonObject != null) {
|
||||
d['U'] = "接口匹配"
|
||||
d['V'] = jsonObject.get("city")
|
||||
d['W'] = jsonObject.get("district")
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class TempExportData {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new TempExportData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
|
||||
for (def city : citys) {
|
||||
def cityRets = getResults(city)
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def getResults(city) {
|
||||
def rets = []
|
||||
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for (def i = 0; i < dists.size(); i++) {
|
||||
def d = dists[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try {
|
||||
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
|
||||
if (objs) {
|
||||
def ret = []
|
||||
ret << d._id.toString()
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
ret << d.address
|
||||
|
||||
def str = ""
|
||||
for (def o : objs) {
|
||||
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
|
||||
}
|
||||
ret << str
|
||||
ret << objs.size()
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println(city +" "+rets.size())
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+544
@@ -0,0 +1,544 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
*/
|
||||
class TownMiddleProcess {
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new TownMiddleProcess();
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
//创建省份
|
||||
// createProvince()
|
||||
//创建城市
|
||||
// createCity()
|
||||
//创建区域
|
||||
// createRegion()
|
||||
//创建乡镇街道
|
||||
createTown()
|
||||
}
|
||||
|
||||
|
||||
def createProvince(){
|
||||
def cat = "省"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||
|
||||
def tjjKeys = tjjMaps.keySet()
|
||||
def mzbKeys = mzbMaps.keySet()
|
||||
|
||||
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||
def lastRets = []
|
||||
tjjMaps.each {key, val->
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", key)
|
||||
udata.put("tjj_province_code", val)
|
||||
udata.put("mzb_province_code", mzbMaps.get(key))
|
||||
udata.put("tjj_source_year",2023)
|
||||
udata.put("mzb_source_year",2023)
|
||||
lastRets << udata
|
||||
}
|
||||
if(diffKeys){
|
||||
diffKeys.each {k->
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", k)
|
||||
if(tjjMaps.get(k)){
|
||||
udata.put("tjj_province_code", tjjMaps.get(k))
|
||||
udata.put("tjj_source_year",2023)
|
||||
}
|
||||
if(mzbMaps.get(k)){
|
||||
udata.put("mzb_province_code", mzbMaps.get(k))
|
||||
udata.put("mzb_source_year",2023)
|
||||
}
|
||||
lastRets << udata
|
||||
}
|
||||
}
|
||||
saveToDB(lastRets, "省份")
|
||||
}
|
||||
|
||||
|
||||
def createCity(){
|
||||
def cat = "城市"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||
def tjjKeys = tjjMaps.keySet()
|
||||
def mzbKeys = mzbMaps.keySet()
|
||||
def sameKeys = tjjKeys.intersect(mzbKeys)
|
||||
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||
def lastRets = []
|
||||
sameKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
lastRets << getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
}
|
||||
|
||||
if(diffKeys){
|
||||
diffKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
lastRets <<getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
}
|
||||
}
|
||||
//添加别名
|
||||
lastRets.each {o->
|
||||
def city = RegionData.formatCity(o.city)
|
||||
if(city){
|
||||
def cityObj = CommonDatautils.findAlCityDetail(city)
|
||||
o.put("alias", cityObj.alias)
|
||||
}else{
|
||||
println o.province+" "+o.city +" 房价网无此城市"
|
||||
}
|
||||
}
|
||||
//添加城市围栏
|
||||
def cityPolylineMap = cityPolyline()
|
||||
//添加pid
|
||||
def provPidMap = createPidData("省份")
|
||||
lastRets.each{o->
|
||||
def shapeData = cityPolylineMap.get(o.city)
|
||||
if(shapeData){
|
||||
o.shape = shapeData.shape
|
||||
o.location = shapeData.location
|
||||
}
|
||||
def provId = provPidMap.get(o.province)
|
||||
if(provId){
|
||||
o.pid = provId
|
||||
}
|
||||
}
|
||||
saveToDB(lastRets, "城市")
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createRegion(){
|
||||
def cat = "区域"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||
def tjjKeys = tjjMaps.keySet()
|
||||
def mzbKeys = mzbMaps.keySet()
|
||||
def sameKeys = tjjKeys.intersect(mzbKeys)
|
||||
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||
def lastRets = []
|
||||
sameKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
// ud.source = "tjj|mzb"
|
||||
lastRets << ud
|
||||
}
|
||||
|
||||
if(diffKeys){
|
||||
diffKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
// if(v){
|
||||
// ud.source = "tjj"
|
||||
// }else{
|
||||
// ud.source = "mzb"
|
||||
// }
|
||||
lastRets << ud
|
||||
}
|
||||
}
|
||||
//添加别名
|
||||
def regionMap = CommonDatautils.getAllCityRegions()
|
||||
lastRets.each {o->
|
||||
def city = RegionData.formatCity(o.city)
|
||||
def district = RegionData.formatRegion(city, o.district)
|
||||
if(city && district){
|
||||
def cMap = regionMap.get(city)
|
||||
if(cMap){
|
||||
def regions = cMap.get(district)
|
||||
o.put("alias", regions)
|
||||
}
|
||||
}else{
|
||||
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
|
||||
}
|
||||
}
|
||||
|
||||
//添加pid
|
||||
def provPidMap = createPidData("城市")
|
||||
|
||||
//添加区域围栏
|
||||
def regionPolylineMap = regionPolyline()
|
||||
|
||||
lastRets.each{o->
|
||||
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
|
||||
if(shapeData){
|
||||
o.shape = shapeData.shape
|
||||
o.location = shapeData.location
|
||||
}
|
||||
def provId = provPidMap.get(o.city)
|
||||
if(provId){
|
||||
o.pid = provId
|
||||
}
|
||||
}
|
||||
//是否为县级市
|
||||
saveToDB(lastRets, "区域")
|
||||
}
|
||||
|
||||
def createTown(){
|
||||
def cat = "乡镇街道"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def lastRets = []
|
||||
tjjMaps.each {k,v->
|
||||
def ud = getAdministrativeCodeInfo(k, v, null)
|
||||
lastRets << ud
|
||||
}
|
||||
|
||||
//添加pid
|
||||
def provPidMap = createPidData("区域")
|
||||
|
||||
//添加区域围栏
|
||||
def regionPolylineMap = townPolyline()
|
||||
|
||||
lastRets.each{o->
|
||||
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
|
||||
if(shapeData){
|
||||
o.shape = shapeData.shape
|
||||
o.location = shapeData.location
|
||||
}
|
||||
def provId = provPidMap.get(o.city)
|
||||
if(provId){
|
||||
o.pid = provId
|
||||
}
|
||||
}
|
||||
//是否为县级市
|
||||
saveToDB(lastRets, "区域")
|
||||
}
|
||||
def getAdministrativeCodeInfo(key, val, mzbMaps){
|
||||
def udata = new BasicDBObject()
|
||||
def keys = key.split("\\|@\\|")
|
||||
def vals = null
|
||||
if(val){
|
||||
vals = val.split("\\|@\\|")
|
||||
}
|
||||
|
||||
def mzb_vals = null
|
||||
if(mzbMaps && mzbMaps.containsKey(key)){
|
||||
def mzb_val_str = mzbMaps.get(key)
|
||||
mzb_vals = mzb_val_str.split("\\|@\\|")
|
||||
}
|
||||
if(keys.size()>=1){
|
||||
udata.put("province", keys[0])
|
||||
if(vals && vals.size()>=1){
|
||||
udata.put("tjj_province_code", vals[0])
|
||||
udata.put("tjj_source_year",2023)
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=1){
|
||||
udata.put("mzb_province_code", mzb_vals[0])
|
||||
udata.put("mzb_source_year",2023)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(keys.size()>=2){
|
||||
udata.put("city", keys[1])
|
||||
if(vals && vals.size()>=2){
|
||||
udata.put("tjj_city_code", vals[1])
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=2){
|
||||
udata.put("mzb_city_code", mzb_vals[1])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(keys.size()>=3){
|
||||
udata.put("district", keys[2])
|
||||
if(vals && vals.size()>=3) {
|
||||
udata.put("tjj_district_code", vals[2])
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=3){
|
||||
udata.put("mzb_district_code", mzb_vals[2])
|
||||
}
|
||||
}
|
||||
return udata
|
||||
}
|
||||
|
||||
|
||||
def saveToDB(lastRets, type){
|
||||
def dataList = []
|
||||
for(def ret : lastRets){
|
||||
ret.put("c_date", new Date())
|
||||
ret.put("m_date", new Date())
|
||||
ret.put("type", type)
|
||||
dataList << ret
|
||||
|
||||
if (dataList.size() >= 5000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
|
||||
dataList.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (dataList.size() > 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
|
||||
dataList.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def createPidData(def type){
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type))
|
||||
def map = [:]
|
||||
def pids = []
|
||||
def step = 2
|
||||
if(type == "城市"){
|
||||
step = 4
|
||||
}else if(type == "区域"){
|
||||
step = 6
|
||||
}
|
||||
for(def d : dbcursor){
|
||||
def pid = null
|
||||
def code = null
|
||||
if(d.mzb_province_code){
|
||||
if(type == "省份") {
|
||||
code = d.mzb_province_code.toString()
|
||||
}else if(type == "城市"){
|
||||
code = d.mzb_city_code.toString()
|
||||
}else if(type == "区域"){
|
||||
code = d.mzb_district_code.toString()
|
||||
}
|
||||
}else if(d.tjj_province_code){
|
||||
if(type == "省份") {
|
||||
code = d.tjj_province_code.toString()
|
||||
}else if(type == "城市"){
|
||||
code = d.tjj_city_code.toString()
|
||||
}else if(ype == "区域"){
|
||||
code = d.tjj_district_code.toString()
|
||||
}
|
||||
}
|
||||
|
||||
pid = code.substring(0,step)
|
||||
if(pids.contains(d.city+"-"+pid)){
|
||||
pid = code
|
||||
}
|
||||
if(!pids.contains(d.city+"-"+pid)){
|
||||
pids << d.city+"-"+pid
|
||||
}
|
||||
if(type == "省份"){
|
||||
map.put(d.province, pid)
|
||||
}else if(type == "城市"){
|
||||
map.put(d.city, pid)
|
||||
}else if(type == "区域"){
|
||||
map.put(d.city+"@"+d.district, pid)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findAllProcincesFromTJJ(cat){
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
def match = null
|
||||
if(cat=="省"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code'))
|
||||
}else if(cat=="城市"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code'))
|
||||
}else if(cat=="区域"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
|
||||
}else if(cat=="乡镇街道"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("name", '$name').append("code", '$code'))
|
||||
}
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
|
||||
// AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(match, group));
|
||||
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city_name").toString()!="省直辖县级行政区划"){
|
||||
def key = obj.get("province_name")
|
||||
def value = obj.get("province_code")
|
||||
if(cat=="城市"){
|
||||
key = key +"|@|"+ obj.get("city_name")
|
||||
value = value +"|@|"+ obj.get("city_code")
|
||||
}else if(cat=="区域"){
|
||||
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
|
||||
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
|
||||
}else if(cat=="乡镇街道"){
|
||||
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("name")
|
||||
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("code")
|
||||
}
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(key, value)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findAllProcincesFromMZB(cat){
|
||||
|
||||
BasicDBObject match = null;
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
|
||||
if(cat == "省"){
|
||||
match = new BasicDBObject('$match', new BasicDBObject("level", "省份"));
|
||||
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code'))
|
||||
}else if(cat == "城市"){
|
||||
match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code'))
|
||||
}else if(cat == "区域"){
|
||||
match = new BasicDBObject('$match', new BasicDBObject("level", "区域"));
|
||||
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code'))
|
||||
}
|
||||
|
||||
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group));
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city").toString()!="省直辖县级行政区划"){
|
||||
def key = obj.get("province")
|
||||
def value = obj.get("mca_province_code")
|
||||
if(cat == "城市"){
|
||||
key = key +"|@|"+obj.get("city")
|
||||
value = value + "|@|" + obj.get("mca_city_code")
|
||||
}else if(cat == "区域"){
|
||||
key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district")
|
||||
value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code")
|
||||
}
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(key, value)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def cityPolyline(){
|
||||
def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject())
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.name+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
map.put(d.name, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def townPolyline(){
|
||||
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+22
-2
@@ -135,6 +135,26 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def format(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
if(address){
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
@@ -228,9 +248,9 @@ class FangjiaClientUtils {
|
||||
def url = null;
|
||||
try {
|
||||
address = address.replace("{","").replace("\\","")
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+18
-18
@@ -33,12 +33,12 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\估家待优化小区 - 副本.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
exportPkPrice()
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
|
||||
|
||||
@@ -153,15 +153,15 @@ class AddPriceTest {
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['M']){
|
||||
def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['M'], cityId)
|
||||
if(d['A']){
|
||||
// def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
|
||||
if(map){
|
||||
d['AK'] = map.avg_price
|
||||
d['AL'] = map.calc_type
|
||||
d['AM'] = map.credit_value
|
||||
// d['AK'] = map.avg_price
|
||||
d['Q'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -227,12 +227,12 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if (!d['K']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['L'] = gxdPrice
|
||||
d['K'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -256,14 +256,14 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(!d['P']){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
d['K'] = dbo.avg_price_2b
|
||||
d['P'] = dbo.avg_price
|
||||
// d['J'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+63
-163
@@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import groovy.json.JsonOutput
|
||||
import net.sf.json.JSONObject
|
||||
import okhttp3.*;
|
||||
import java.io.IOException;
|
||||
@@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit;
|
||||
import okhttp3.Request.Builder;
|
||||
|
||||
class QiliangTest {
|
||||
String proxyHost = 'tunnel2.qg.net';
|
||||
Integer proxyPort = 14186;
|
||||
String proxyUsername = '113BBC06';
|
||||
String proxyPassword = '3B2FBD05B0AF';
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
|
||||
// static main(args) {
|
||||
// def task = new QiliangTest()
|
||||
//// task.run()
|
||||
// task.getHttpResult("扬州扬子江北路399号11041")
|
||||
// }
|
||||
static main(args) {
|
||||
QiliangTest task = new QiliangTest();
|
||||
// task.run()
|
||||
|
||||
def run(){//地址测试-2024.3.4-结果.xlsx
|
||||
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
||||
if(!row['N']){
|
||||
def address = null
|
||||
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
||||
address = row['B']+row['C']
|
||||
}else{
|
||||
address = row['C']
|
||||
}
|
||||
if(address){
|
||||
def result = getHttpResult(address)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['N'] = data.province
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['O'] = data.city
|
||||
}
|
||||
if(data.district && data.district!='null'){
|
||||
row['P'] = data.district
|
||||
}
|
||||
if(data.town && data.town!='null'){
|
||||
row['Q'] = data.town
|
||||
}
|
||||
if(data.village && data.village!='null'){
|
||||
row['R'] = data.village
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['S'] = data.areaCode
|
||||
}
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['T'] = "非农"
|
||||
}else{
|
||||
row['T'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['U'] = "否"
|
||||
}else{
|
||||
row['U'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['V'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['W'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
def data = [
|
||||
'address': '浙江省湖州市德清县莫干山开发区中小工业园区',
|
||||
'type': '2'
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
}
|
||||
|
||||
def getHttpResult(address){
|
||||
def data = new JSONObject()
|
||||
data.put("address", address)
|
||||
data.put("type","2")
|
||||
|
||||
Map headers = new HashMap();
|
||||
headers.put("Accept", "application/json, text/plain, */*");
|
||||
headers.put("Accept-Encoding", "gzip, deflate");
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
headers.put("Content-Type","application/json;charset=UTF-8")
|
||||
headers.put("Host","146.56.217.44:9010")
|
||||
headers.put("Origin","http://146.56.217.44:8088")
|
||||
headers.put("Proxy-Connection","keep-alive")
|
||||
headers.put("Referer","http://146.56.217.44:8088/")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||
|
||||
HttpRequest request1 = new HttpRequest()
|
||||
try{
|
||||
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println response
|
||||
return response
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
|
||||
String getProxyResult(data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
String jsonData = new JsonOutput().toJson(data);
|
||||
// 创建代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
|
||||
// 创建 URL 对象
|
||||
URL url = new URL(targetUrl);
|
||||
|
||||
// 创建连接对象
|
||||
URLConnection connection = url.openConnection(proxy);
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST");
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true);
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json");
|
||||
|
||||
// 发送请求参数
|
||||
OutputStream outputStream = connection.getOutputStream();
|
||||
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 读取响应
|
||||
int responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
final static String proxyIp = "tunnel2.qg.net";
|
||||
final static Integer proxyPort = 14186;
|
||||
final static String authKey = "113BBC06";
|
||||
final static String password = "3B2FBD05B0AF";
|
||||
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
|
||||
|
||||
// public String request(params) throws IOException {
|
||||
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
|
||||
// OkHttpClient client = new OkHttpClient.Builder()
|
||||
// .proxy(proxy)
|
||||
// .proxyAuthenticator((route, response){
|
||||
// String credential = Credentials.basic(authKey, password);
|
||||
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
|
||||
// }).
|
||||
// build();
|
||||
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
|
||||
//
|
||||
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
|
||||
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
|
||||
//
|
||||
// Response response = client.newCall(request).execute();
|
||||
// return response.body().string();
|
||||
//
|
||||
//// return client.newCall(request).execute();
|
||||
// }
|
||||
public static void main(String[] args) {
|
||||
def proxyHost = "tunnel2.qg.net"
|
||||
def proxyPort = 14186
|
||||
def proxyUsername = '113BBC06'
|
||||
def proxyPassword = '3B2FBD05B0AF'
|
||||
|
||||
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
|
||||
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
|
||||
|
||||
|
||||
// 创建代理服务器的连接
|
||||
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
|
||||
|
||||
// 打开代理连接
|
||||
def proxyConnection = proxy.openConnection()
|
||||
|
||||
// 创建目标URL的连接
|
||||
def url = new URL(targetUrl)
|
||||
def connection = url.openConnection(proxy)
|
||||
|
||||
// 设置请求方法
|
||||
connection.setRequestMethod("GET")
|
||||
|
||||
// 添加请求头(可选)
|
||||
connection.setRequestProperty("Content-Type", "application/json")
|
||||
|
||||
// 构建查询参数
|
||||
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
|
||||
|
||||
// 将查询参数附加到目标URL
|
||||
def finalUrl = "${targetUrl}?${queryString}"
|
||||
|
||||
// 发送请求
|
||||
connection.connect()
|
||||
|
||||
// 获取响应
|
||||
def responseCode = connection.getResponseCode()
|
||||
def responseMessage = connection.getResponseMessage()
|
||||
|
||||
// 读取响应内容
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
||||
def responseContent = reader.readLine()
|
||||
reader.close()
|
||||
|
||||
println "Response Code: ${responseCode}"
|
||||
println "Response Message: ${responseMessage}"
|
||||
println "Response Content: ${responseContent}"
|
||||
}
|
||||
}
|
||||
|
||||
+11
-11
@@ -19,17 +19,17 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
|
||||
//def s = getTownshipInfo("118.702988,39.740979")
|
||||
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
// if(!row['I']){
|
||||
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
// if(rets){
|
||||
// def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
// if(obj){
|
||||
// row['I'] = obj.lng
|
||||
// row['J'] = obj.lat
|
||||
// println row['C'] + " "+obj
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
if(!row['I'] && row['I']=='宝应县'){
|
||||
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
if(rets){
|
||||
def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
if(obj){
|
||||
row['I'] = obj.lng
|
||||
row['J'] = obj.lat
|
||||
println row['C'] + " "+obj
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(row['D'] && !row['F']){
|
||||
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import groovy.json.JsonOutput
|
||||
import net.sf.json.JSONObject
|
||||
import okhttp3.*;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Proxy;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
||||
import okhttp3.Request.Builder;
|
||||
|
||||
class QiliangTest1 {
|
||||
def proxyHost = 'tunnel2.qg.net'
|
||||
def proxyPort = 14186
|
||||
def proxyUsername = '113BBC06'
|
||||
def proxyPassword = '3B2FBD05B0AF'
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
|
||||
static main(args) {
|
||||
def task = new QiliangTest()
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
|
||||
'type': '1'
|
||||
]
|
||||
def s = task.getProxyResult(data)
|
||||
print(s)
|
||||
}
|
||||
|
||||
def run(){//地址测试-2024.3.4-结果.xlsx
|
||||
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
||||
if(!row['N']){
|
||||
def address = null
|
||||
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
||||
address = row['B']+row['C']
|
||||
}else{
|
||||
address = row['C']
|
||||
}
|
||||
if(address){
|
||||
def result = getHttpResult(address)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['N'] = data.province
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['O'] = data.city
|
||||
}
|
||||
if(data.district && data.district!='null'){
|
||||
row['P'] = data.district
|
||||
}
|
||||
if(data.town && data.town!='null'){
|
||||
row['Q'] = data.town
|
||||
}
|
||||
if(data.village && data.village!='null'){
|
||||
row['R'] = data.village
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['S'] = data.areaCode
|
||||
}
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['T'] = "非农"
|
||||
}else{
|
||||
row['T'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['U'] = "否"
|
||||
}else{
|
||||
row['U'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['V'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['W'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
JSONObject getHttpResult(address){
|
||||
JSONObject data = new JSONObject()
|
||||
data.put("address", address)
|
||||
data.put("type","2")
|
||||
|
||||
Map headers = new HashMap();
|
||||
headers.put("Accept", "application/json, text/plain, */*");
|
||||
headers.put("Accept-Encoding", "gzip, deflate");
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
headers.put("Content-Type","application/json;charset=UTF-8")
|
||||
headers.put("Host","146.56.217.44:9010")
|
||||
headers.put("Origin","http://146.56.217.44:8088")
|
||||
headers.put("Proxy-Connection","keep-alive")
|
||||
headers.put("Referer","http://146.56.217.44:8088/")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||
|
||||
HttpRequest request1 = new HttpRequest()
|
||||
try{
|
||||
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
||||
JSONObject response = JSONObject.fromObject(res)
|
||||
println response
|
||||
return response
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def getProxyResult(data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
def jsonData = new JsonOutput().toJson(data)
|
||||
// 创建代理对象
|
||||
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
|
||||
|
||||
// 创建 URL 对象
|
||||
def url = new URL(targetUrl)
|
||||
|
||||
// 创建连接对象
|
||||
def connection = url.openConnection(proxy)
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST")
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true)
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json")
|
||||
|
||||
// 发送请求参数
|
||||
def outputStream = connection.getOutputStream()
|
||||
outputStream.write(jsonData.getBytes("UTF-8"))
|
||||
outputStream.flush()
|
||||
outputStream.close()
|
||||
|
||||
// 读取响应
|
||||
def responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
||||
def response = new StringBuilder()
|
||||
String line
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line)
|
||||
}
|
||||
reader.close()
|
||||
return response.toString()
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}"
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
+17
-16
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
@@ -34,16 +35,18 @@ class PoiWash {
|
||||
public static def regions = []
|
||||
public static def nameKeys = null
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
// task.updateId()
|
||||
// task.run()
|
||||
// task.synch()
|
||||
// task.check()
|
||||
task.deduplicate()
|
||||
// task.deduplicate()
|
||||
// task.updateEsMapPoi()
|
||||
// task.checkSpecialSymbols()
|
||||
// task.splitRoad()
|
||||
task.splitRoad()
|
||||
}
|
||||
|
||||
def updateId(){
|
||||
@@ -452,9 +455,10 @@ class PoiWash {
|
||||
|
||||
def splitRoad(){
|
||||
//"三明市",
|
||||
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys) {
|
||||
def datas = findPoiByQuery(city, ['city','address'])
|
||||
def cityObj = fullCityMap.get(city)
|
||||
def datas = findPoiByQuery(cityObj.city, ['city','address'])
|
||||
splitRoadRun(datas)
|
||||
println city +" 完成"
|
||||
}
|
||||
@@ -472,18 +476,15 @@ class PoiWash {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
|
||||
if(jsonObject && jsonObject.containsKey("result")){
|
||||
def dataMap = jsonObject.get("result")
|
||||
if(dataMap.get("street")){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street", dataMap.get("street"))
|
||||
if(dataMap.get("streetNumber")){
|
||||
udata.put("street_number", dataMap.get("streetNumber"))
|
||||
}
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
|
||||
udata.put("m_date", new Date())
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
|
||||
} catch (Exception e) {
|
||||
|
||||
+31
-22
@@ -26,8 +26,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
||||
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
||||
@@ -40,7 +40,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
@@ -58,10 +58,11 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
run()
|
||||
// run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
//codeRun()
|
||||
codeRun()
|
||||
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
@@ -346,24 +347,32 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def codeRun(){
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// def dbcursor = mca_administrative_division_coll.find(q)
|
||||
// def rets = []
|
||||
// for(def it : dbcursor){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("_id", it._id)
|
||||
// udata.put("province", it.province)
|
||||
// udata.put("city", it.city)
|
||||
// udata.put("district", it.district)
|
||||
// udata.put("mca_province_code", it.province_code)
|
||||
// udata.put("mca_city_code", it.city_code)
|
||||
// udata.put("mca_district_code", it.district_code)
|
||||
// udata.put("level", it.level)
|
||||
// udata.put("c_date", it.c_date)
|
||||
// rets << udata
|
||||
//// administrative_division_100_CONN.save(udata)
|
||||
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
// }
|
||||
// administrative_division_100_CONN.insert(rets)
|
||||
//
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = mca_administrative_division_coll.find(q)
|
||||
def rets = []
|
||||
for(def it : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("_id", it._id)
|
||||
udata.put("province", it.province)
|
||||
udata.put("city", it.city)
|
||||
udata.put("district", it.district)
|
||||
udata.put("mca_province_code", it.province_code)
|
||||
udata.put("mca_city_code", it.city_code)
|
||||
udata.put("mca_district_code", it.district_code)
|
||||
udata.put("level", it.level)
|
||||
udata.put("c_date", it.c_date)
|
||||
rets << udata
|
||||
// administrative_division_100_CONN.save(udata)
|
||||
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
q.put("province","°ÄÃÅÌØ±ðÐÐÕþÇø")
|
||||
def dbcursor = administrative_division_2022_CONN.find(q)
|
||||
for(def d : dbcursor){
|
||||
administrative_division_100_CONN.save(d)
|
||||
|
||||
}
|
||||
administrative_division_100_CONN.insert(rets)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
|
||||
* 8:華硊摯華硊梗靡脤梑
|
||||
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
|
||||
* 10:⑽秞⑤饜
|
||||
*11:open½Ó¿ÚÆ¥Åä
|
||||
* 11:open½Ó¿ÚÆ¥Åä
|
||||
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
|
||||
Reference in New Issue
Block a user