update
This commit is contained in:
+36
-34
@@ -1289,52 +1289,54 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格不作为指导价录入5-20.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['J']){
|
||||
//// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//// if(dist){
|
||||
//// d['G'] = dist.region
|
||||
//// d['H'] = dist._id.toString()
|
||||
//// }
|
||||
//// }
|
||||
// if(!d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['G']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
// if(dist){
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['H'] && d['G'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
if(d['B']){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['B'])
|
||||
if(obj){
|
||||
def markMap = obj.mark_map
|
||||
if(!obj.mark_map){
|
||||
markMap = [:]
|
||||
d['H'] = dist.name
|
||||
}
|
||||
def data = new BasicDBObject()
|
||||
markMap.put("exclusive", "是")
|
||||
data.put("mark_map", markMap)
|
||||
data.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
})
|
||||
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
// if(d['B']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(obj){
|
||||
// def markMap = obj.mark_map
|
||||
// if(!obj.mark_map){
|
||||
// markMap = [:]
|
||||
// }
|
||||
// def data = new BasicDBObject()
|
||||
// markMap.put("exclusive", "是")
|
||||
// data.put("mark_map", markMap)
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
|
||||
+74
-22
@@ -29,10 +29,10 @@ class DevProcess {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
// task.devSplitRoad()
|
||||
task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
task.matchDev()
|
||||
// task.matchDev()
|
||||
|
||||
// task.calAddrDistince()
|
||||
// task.updateCityAndDist()
|
||||
@@ -46,6 +46,9 @@ class DevProcess {
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
}
|
||||
|
||||
|
||||
@@ -165,10 +168,10 @@ class DevProcess {
|
||||
|
||||
def devSplitRoad(){
|
||||
def i = 0;
|
||||
while(i<1){
|
||||
def fileName = "/home/goojia/gj/company-${i}.xlsx"
|
||||
while(i<14){
|
||||
def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\56万企业地址\\省没出值地方.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\浙江\\zhejiang_0.xlsx"
|
||||
}
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
/*if(d['C'] && !d['V']){
|
||||
@@ -194,7 +197,7 @@ class DevProcess {
|
||||
}
|
||||
}
|
||||
}*/
|
||||
if(d['C'] && d['B']!='河北省' && !d['J']){
|
||||
if(d['C'] && !d['J']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
if(jsonObject){
|
||||
def street = jsonObject.get("street")
|
||||
@@ -292,10 +295,10 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上.xlsx",{d->
|
||||
if(!d['F']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['D'].replace("河北", "").replace("浙江",""))
|
||||
q.put("name", d['C'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['F'] = obj._id.toString()
|
||||
@@ -507,6 +510,27 @@ class DevProcess {
|
||||
})
|
||||
}
|
||||
|
||||
def addNewDev2(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
|
||||
if(d['F'] && d['F']=="新增"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.create_time = new Date()
|
||||
udata.province = d['G']
|
||||
udata.city = d['I']
|
||||
udata.district = d['J']
|
||||
udata.level = "省级"
|
||||
udata.name = d['D']
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
|
||||
if(!obj){
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
println udata.name +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createDevMapping(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
for(def d : dbcursor){
|
||||
@@ -550,15 +574,15 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def deleteAddr2(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北开发区删除地址.xlsx",{d->
|
||||
if(d['E']){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d->
|
||||
if(d['Q'] && d['M']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['E']))
|
||||
q.put("_id", new ObjectId(d['Q']))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def q2 = new BasicDBObject()
|
||||
q2.put("name", obj.name)
|
||||
q2.put("address", d['A']+d['B'])
|
||||
q2.put("address", d['I']+d['J'])
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q2)
|
||||
if(sameObj){
|
||||
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id))
|
||||
@@ -580,17 +604,19 @@ class DevProcess {
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万企业地址\\56万需修改地址.xlsx",{d->
|
||||
if(d['L'] && d['L']=="路正常" && d['K']){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上(49个对应名称1个新增).xlsx",{d->
|
||||
if(d['G']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['K'])
|
||||
q.put("name", d['E'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
def address = d['I']+d['J']
|
||||
q.put("address", address)
|
||||
d['N'] = obj._id.toString()
|
||||
d['O'] = obj.name
|
||||
// def address = d['F']//d['I']+d['J']
|
||||
// q.put("address", address)
|
||||
def street = d['G']
|
||||
q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -603,13 +629,39 @@ class DevProcess {
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address", address)
|
||||
// udata.put("type", "地址")
|
||||
udata.put("type", "乡镇街道")
|
||||
udata.put("township", street)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+address
|
||||
println obj.name +" "+street
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def addAlias(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['D'] = obj._id.toString()
|
||||
def arrs = d['B'].split("、")
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+26
-26
@@ -13,7 +13,7 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\202311广州.xlsx"
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
@@ -69,21 +69,21 @@ class MsbPocTest {
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['S'] && row['D']){
|
||||
if(!row['Y'] && row['G']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['H']
|
||||
def region = null //row['D']
|
||||
def name = row['E']
|
||||
def address = row['G']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['S'] = jsonObject.get("id")
|
||||
row['T'] = jsonObject.get("city")
|
||||
row['U'] = jsonObject.get("district")
|
||||
row['V'] = jsonObject.get("name")
|
||||
row['W'] = jsonObject.get("propertyType")
|
||||
row['X'] = jsonObject.get("credit")
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("propertyType")
|
||||
// row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -111,18 +111,18 @@ class MsbPocTest {
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['N']){
|
||||
def city = row['A']
|
||||
def region = row['B']
|
||||
def address = row['C'] //row['G']//row['F']
|
||||
if(!row['Y']){
|
||||
def city = row['C']
|
||||
def region = null //row['B']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
if(jsonObject != null) {
|
||||
row['N'] = jsonObject.get("id")
|
||||
row['O'] = jsonObject.get("city")
|
||||
row['P'] = jsonObject.get("district")
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -169,17 +169,17 @@ class MsbPocTest {
|
||||
def addCCBPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['H']){
|
||||
if(!row['V']){
|
||||
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
// if(dist){
|
||||
if(row['D'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['D'])
|
||||
if(row['Y'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['Y'])
|
||||
if(d){
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row['H'] = map.avg_price
|
||||
row['I'] = map.calc_type
|
||||
row['V'] = map.avg_price
|
||||
// row['I'] = map.calc_type
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -33,14 +33,14 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
addTradePrice(filePath)
|
||||
// addTradePrice(filePath)
|
||||
|
||||
|
||||
}
|
||||
|
||||
+138
-73
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -39,7 +41,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
|
||||
@@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
@@ -78,29 +80,38 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
// sysnchVillage()
|
||||
sysnchVillage()
|
||||
|
||||
addTownship()
|
||||
// addTownship()
|
||||
}
|
||||
|
||||
def addTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d->
|
||||
if(d['J']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d->
|
||||
if(d['E']){
|
||||
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// d['H'] = gaodeMap.city
|
||||
// d['I'] = gaodeMap.district
|
||||
// d['J'] = gaodeMap.township
|
||||
// d['K'] = gaodeMap.format_addr
|
||||
// if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
// d['H'] = gaodeMap.city
|
||||
// }
|
||||
// if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
// d['I'] = gaodeMap.district
|
||||
// }
|
||||
// if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
// d['J'] = gaodeMap.township
|
||||
// }
|
||||
// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
// d['K'] = gaodeMap.format_addr
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['H'])
|
||||
q.put("district", d['I'])
|
||||
q.put("street",d['G'])
|
||||
q.put("street",d['J'])
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
|
||||
if(obj){
|
||||
d['O'] = "是"
|
||||
d['L'] = "是"
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -108,107 +119,161 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def sysnchVillage(){
|
||||
def pois = findAllVillagePoiMap()
|
||||
def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"]
|
||||
for(def city : buducitys){
|
||||
def pois = findAllVillagePoiMap(city)
|
||||
def rows = createVillageData(city, pois)
|
||||
|
||||
save(city, rows)
|
||||
saveToMG(city, rows)
|
||||
println city +" "+pois.size()+" add:"+rows.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def createVillageData(city, pois){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
|
||||
|
||||
def townMap = findTownRange(city)
|
||||
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = MAPPOI_CONN.findOne(new BasicDBObject("city", p.city).append("district", p.district).append("name", p.name))
|
||||
if(!obj){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
def townMap = findTownRange(p.city)
|
||||
createVillageData(prov, p, townMap)
|
||||
}
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
|
||||
if(!obj){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
def ret = createVillageData(prov, p, townMap)
|
||||
rets.addAll(ret)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def isTown( city, region, name){
|
||||
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
|
||||
// obj = it
|
||||
// },"192.168.0.89")
|
||||
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool":{
|
||||
"must":[
|
||||
{"term": {"city.keyword":"${city}"}},
|
||||
{"term": {"district.keyword":"${region}"}},
|
||||
{"term": {"town.keyword":"${town}"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
def obj = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
obj = hits[0]
|
||||
}
|
||||
}
|
||||
println(response)
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def createVillageData(province, map, townMap){
|
||||
def fjw_province = province
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
// def fjw_province = province
|
||||
// def fjw_city = RegionData.formatCity(map.city)
|
||||
// def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
|
||||
map.province = ProvinceData.getFullProvince(province)
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
def retMap = [:]
|
||||
retMap.province = ProvinceData.getFullProvince(province)
|
||||
// retMap.fjw_province = fjw_province
|
||||
// retMap.fjw_city = fjw_city
|
||||
|
||||
if(map.cat){
|
||||
map.type = "地名地址信息"
|
||||
map.secondary_type = "普通地名"
|
||||
map.small_type = "村庄级地名"
|
||||
map.cat = "地名地址信息;普通地名;村庄级地名"
|
||||
retMap.city = map.city
|
||||
retMap.district = map.district
|
||||
retMap.name = map.name
|
||||
retMap.type = "地名地址信息"
|
||||
retMap.secondary_type = "普通地名"
|
||||
retMap.small_type = "村庄级地名"
|
||||
retMap.cat = "地名地址信息;普通地名;村庄级地名"
|
||||
|
||||
}
|
||||
map.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
|
||||
retMap.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
|
||||
retMap.township = InsidePolygon.isInArea2(retMap.location.lon, retMap.location.lat, townMap)
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(retMap.location.lat, retMap.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
retMap.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(retMap.location.lat, retMap.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
retMap.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!map.township){
|
||||
if(!retMap.township){
|
||||
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
def gaodeMap = getTownshipInfo(retMap.location.lon+","+retMap.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
retMap.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
// if(!fjw_region){
|
||||
// if(gaodeMap && gaodeMap.district){
|
||||
// fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(!fjw_region){
|
||||
// println fjw_city+" "+retMap.region+" 无法格式化 "+retMap.location + " "+gaodeMap
|
||||
// }
|
||||
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
|
||||
map.fjw_district = fjw_region
|
||||
// retMap.fjw_district = fjw_region
|
||||
}
|
||||
map.fjw_district = fjw_region
|
||||
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
map.version = version
|
||||
// retMap.fjw_district = fjw_region
|
||||
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(retMap.name, " ")
|
||||
retMap.version = version
|
||||
retMap.data_source = "百度地图"
|
||||
retMap.data_id = map._id.toString()
|
||||
retMap.id = map._id.toString()
|
||||
|
||||
if(map.secondary_type && TAGS.contains(map.secondary_type)){
|
||||
map.status = 0
|
||||
if(retMap.secondary_type && TAGS.contains(retMap.secondary_type)){
|
||||
retMap.status = 0
|
||||
}else{
|
||||
map.status = 1
|
||||
retMap.status = 1
|
||||
}
|
||||
return map
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findAllVillagePoiMap(){
|
||||
def findAllVillagePoiMap(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
@@ -216,7 +281,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
keys.put("addr", 1)
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄"), keys).limit(10).toList()
|
||||
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄").append("city", city), keys).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
@@ -490,7 +555,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def client = ElasticUtils.getClient("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
+17
-2
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -58,15 +60,16 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
// run()
|
||||
run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
codeRun()
|
||||
// codeRun()
|
||||
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
// addTest()
|
||||
// updateTownship()
|
||||
}
|
||||
|
||||
def addTest(){
|
||||
@@ -375,4 +378,16 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updateTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
|
||||
if(row['Q'] && row['Q']=='是'){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", row['J'])
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ class ElasticUtils {
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));//goojia123456
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user