Merge branch 'script_gj' into 'master'

update

See merge request !848
This commit is contained in:
2024-05-26 00:07:57 +08:00
7 changed files with 295 additions and 161 deletions
@@ -1289,52 +1289,54 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格不作为指导价录入5-20.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['B'], d['C'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//// if(dist){
//// d['G'] = dist.region
//// d['H'] = dist._id.toString()
//// }
//// }
// if(!d['A']){
// def dist = CommonDatautils.findDistrictById(d['J'])
def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
// if(dist){
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// }
// }
if(!d['H'] && d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
// }
// }
// }
//},0)
def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['B']){
def obj = CommonDatautils.findDistrictById(d['B'])
if(obj){
def markMap = obj.mark_map
if(!obj.mark_map){
markMap = [:]
d['H'] = dist.name
}
def data = new BasicDBObject()
markMap.put("exclusive", "是")
data.put("mark_map", markMap)
data.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
}
}
},0)
})
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
//ExcelUtils.multiThreadingModify(filePath, { d ->
// if(d['B']){
//
// def obj = CommonDatautils.findDistrictById(d['B'])
// if(obj){
// def markMap = obj.mark_map
// if(!obj.mark_map){
// markMap = [:]
// }
// def data = new BasicDBObject()
// markMap.put("exclusive", "是")
// data.put("mark_map", markMap)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
@@ -29,10 +29,10 @@ class DevProcess {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
// task.devSplitRoad()
task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
task.matchDev()
// task.matchDev()
// task.calAddrDistince()
// task.updateCityAndDist()
@@ -46,6 +46,9 @@ class DevProcess {
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo43()
// task.addNewDev2()
// task.addAlias()
}
@@ -165,10 +168,10 @@ class DevProcess {
def devSplitRoad(){
def i = 0;
while(i<1){
def fileName = "/home/goojia/gj/company-${i}.xlsx"
while(i<14){
def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\56万企业地址\\省没出值地方.xlsx"
fileName = "D:\\数据\\数据建设\\开发区\\浙江\\zhejiang_0.xlsx"
}
ExcelUtils.multiThreadingModify(fileName,{d->
/*if(d['C'] && !d['V']){
@@ -194,7 +197,7 @@ class DevProcess {
}
}
}*/
if(d['C'] && d['B']!='河北省' && !d['J']){
if(d['C'] && !d['J']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(jsonObject){
def street = jsonObject.get("street")
@@ -292,10 +295,10 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上.xlsx",{d->
if(!d['F']){
def q = new BasicDBObject()
q.put("name", d['D'].replace("河北", "").replace("浙江",""))
q.put("name", d['C'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['F'] = obj._id.toString()
@@ -507,6 +510,27 @@ class DevProcess {
})
}
def addNewDev2(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
if(d['F'] && d['F']=="新增"){
def udata = new BasicDBObject()
udata.create_time = new Date()
udata.province = d['G']
udata.city = d['I']
udata.district = d['J']
udata.level = "省级"
udata.name = d['D']
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
if(!obj){
development_zone_CONN.save(udata)
}else{
println udata.name +" 已存在"
}
}
})
}
def createDevMapping(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
for(def d : dbcursor){
@@ -550,15 +574,15 @@ class DevProcess {
}
def deleteAddr2(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北开发区删除地址.xlsx",{d->
if(d['E']){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d->
if(d['Q'] && d['M']){
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['E']))
q.put("_id", new ObjectId(d['Q']))
def obj = development_zone_CONN.findOne(q)
if(obj){
def q2 = new BasicDBObject()
q2.put("name", obj.name)
q2.put("address", d['A']+d['B'])
q2.put("address", d['I']+d['J'])
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(sameObj){
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id))
@@ -580,17 +604,19 @@ class DevProcess {
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万企业地址\\56万需修改地址.xlsx",{d->
if(d['L'] && d['L']=="路正常" && d['K']){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上(49个对应名称1个新增).xlsx",{d->
if(d['G']){
def q = new BasicDBObject()
q.put("name", d['K'])
q.put("name", d['E'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['Q'] = obj._id.toString()
d['R'] = obj.name
def address = d['I']+d['J']
q.put("address", address)
d['N'] = obj._id.toString()
d['O'] = obj.name
// def address = d['F']//d['I']+d['J']
// q.put("address", address)
def street = d['G']
q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){
def udata = new BasicDBObject()
@@ -603,13 +629,39 @@ class DevProcess {
}
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address", address)
// udata.put("type", "地址")
udata.put("type", "乡镇街道")
udata.put("township", street)
DEV_MAPPING_CONN.save(udata)
println obj.name +" "+address
println obj.name +" "+street
}
}
}
},0);
}
def addAlias(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
def q = new BasicDBObject()
q.put("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['D'] = obj._id.toString()
def arrs = d['B'].split("、")
def alias = obj.alias
if(!alias){
alias = []
}
for(def s : arrs){
if(!alias.contains(s)){
alias << s
}
}
def udata = new BasicDBObject()
udata.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
})
}
}
@@ -13,7 +13,7 @@ class MsbPocTest {
task.run();
}
def run(){
def inputfile = "D:\\数据\\建行\\房产估值\\202311广州.xlsx"
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
@@ -69,21 +69,21 @@ class MsbPocTest {
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['S'] && row['D']){
if(!row['Y'] && row['G']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['H']
def region = null //row['D']
def name = row['E']
def address = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['S'] = jsonObject.get("id")
row['T'] = jsonObject.get("city")
row['U'] = jsonObject.get("district")
row['V'] = jsonObject.get("name")
row['W'] = jsonObject.get("propertyType")
row['X'] = jsonObject.get("credit")
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("propertyType")
// row['X'] = jsonObject.get("credit")
}
}
})
@@ -111,18 +111,18 @@ class MsbPocTest {
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['N']){
def city = row['A']
def region = row['B']
def address = row['C'] //row['G']//row['F']
if(!row['Y']){
def city = row['C']
def region = null //row['B']
def address = row['G'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null) {
row['N'] = jsonObject.get("id")
row['O'] = jsonObject.get("city")
row['P'] = jsonObject.get("district")
row['Q'] = jsonObject.get("name")
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
}
}
})
@@ -169,17 +169,17 @@ class MsbPocTest {
def addCCBPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['H']){
if(!row['V']){
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
// if(dist){
if(row['D'] instanceof String){
def d = CommonDatautils.findDistrictById(row['D'])
if(row['Y'] instanceof String){
def d = CommonDatautils.findDistrictById(row['Y'])
if(d){
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
if(map){
row['H'] = map.avg_price
row['I'] = map.calc_type
row['V'] = map.avg_price
// row['I'] = map.calc_type
}
}
}
@@ -33,14 +33,14 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
addTradePrice(filePath)
// addTradePrice(filePath)
}
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -39,7 +41,7 @@ import java.util.concurrent.atomic.AtomicLong
@BatchOwner(Watcher.GaoJuan)
class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static def version = new Date().getTime()
@@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
@@ -78,29 +80,38 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
// sysnchVillage()
sysnchVillage()
addTownship()
// addTownship()
}
def addTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d->
if(d['J']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d->
if(d['E']){
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
// if(gaodeMap && gaodeMap.township){
// d['H'] = gaodeMap.city
// d['I'] = gaodeMap.district
// d['J'] = gaodeMap.township
// d['K'] = gaodeMap.format_addr
// if(gaodeMap.city && gaodeMap.city!="[]"){
// d['H'] = gaodeMap.city
// }
// if( gaodeMap.district && gaodeMap.district!='[]'){
// d['I'] = gaodeMap.district
// }
// if(gaodeMap.township && gaodeMap.township!='[]'){
// d['J'] = gaodeMap.township
// }
// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
// d['K'] = gaodeMap.format_addr
// }
//
// }
def q = new BasicDBObject()
q.put("city", d['H'])
q.put("district", d['I'])
q.put("street",d['G'])
q.put("street",d['J'])
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
if(obj){
d['O'] = "是"
d['L'] = "是"
}
}
})
@@ -108,107 +119,161 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def pois = findAllVillagePoiMap()
def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"]
for(def city : buducitys){
def pois = findAllVillagePoiMap(city)
def rows = createVillageData(city, pois)
save(city, rows)
saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
}
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
def townMap = findTownRange(city)
for(def i=0;i<pois.size();i++){
def p = pois[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def obj = MAPPOI_CONN.findOne(new BasicDBObject("city", p.city).append("district", p.district).append("name", p.name))
if(!obj){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def townMap = findTownRange(p.city)
createVillageData(prov, p, townMap)
}
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
if(!obj){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
def ret = createVillageData(prov, p, townMap)
rets.addAll(ret)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def isTown( city, region, name){
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
// obj = it
// },"192.168.0.89")
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
def body = """
{
"query": {
"bool":{
"must":[
{"term": {"city.keyword":"${city}"}},
{"term": {"district.keyword":"${region}"}},
{"term": {"town.keyword":"${town}"}}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
def obj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
obj = hits[0]
}
}
println(response)
return obj
}
def createVillageData(province, map, townMap){
def fjw_province = province
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
// def fjw_province = province
// def fjw_city = RegionData.formatCity(map.city)
// def fjw_region = RegionData.formatRegion(fjw_city, map.district)
map.province = ProvinceData.getFullProvince(province)
map.fjw_province = fjw_province
map.fjw_city = fjw_city
def retMap = [:]
retMap.province = ProvinceData.getFullProvince(province)
// retMap.fjw_province = fjw_province
// retMap.fjw_city = fjw_city
if(map.cat){
map.type = "地名地址信息"
map.secondary_type = "普通地名"
map.small_type = "村庄级地名"
map.cat = "地名地址信息;普通地名;村庄级地名"
retMap.city = map.city
retMap.district = map.district
retMap.name = map.name
retMap.type = "地名地址信息"
retMap.secondary_type = "普通地名"
retMap.small_type = "村庄级地名"
retMap.cat = "地名地址信息;普通地名;村庄级地名"
}
map.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
retMap.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
retMap.township = InsidePolygon.isInArea2(retMap.location.lon, retMap.location.lat, townMap)
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
def baiduMap = GeoUtils.gcj02ToBd09(retMap.location.lat, retMap.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
map.bd_location = location
retMap.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
def gpsLoactionMap = GeoUtils.gcjToGps84(retMap.location.lat, retMap.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
map.gps_location = gpslocation
retMap.gps_location = gpslocation
}
if(!map.township){
if(!retMap.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
def gaodeMap = getTownshipInfo(retMap.location.lon+","+retMap.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
retMap.township = gaodeMap.township
}
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
}
}
// if(!fjw_region){
// if(gaodeMap && gaodeMap.district){
// fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
// }
// }
//
// if(!fjw_region){
// println fjw_city+" "+retMap.region+" 无法格式化 "+retMap.location + " "+gaodeMap
// }
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_district = fjw_region
// retMap.fjw_district = fjw_region
}
map.fjw_district = fjw_region
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
map.version = version
// retMap.fjw_district = fjw_region
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(retMap.name, " ")
retMap.version = version
retMap.data_source = "百度地图"
retMap.data_id = map._id.toString()
retMap.id = map._id.toString()
if(map.secondary_type && TAGS.contains(map.secondary_type)){
map.status = 0
if(retMap.secondary_type && TAGS.contains(retMap.secondary_type)){
retMap.status = 0
}else{
map.status = 1
retMap.status = 1
}
return map
return retMap
}
def findAllVillagePoiMap(){
def findAllVillagePoiMap(city){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
@@ -216,7 +281,7 @@ class GaoDePoiDataProcess extends BaseBatch{
keys.put("addr", 1)
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄"), keys).limit(10).toList()
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄").append("city", city), keys).toList()
return list;
}
@@ -490,7 +555,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def client = ElasticUtils.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
@@ -58,15 +60,16 @@ class SeaweedDataSynch extends BaseBatch{
public void process(Object args) {
// run()
run()
// println "全国district, shop, office同步完成"
//民政部编码上传
codeRun()
// codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
// addTest()
// updateTownship()
}
def addTest(){
@@ -375,4 +378,16 @@ class SeaweedDataSynch extends BaseBatch{
}
}
def updateTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
if(row['Q'] && row['Q']=='是'){
def udata = new BasicDBObject()
udata.put("township", row['J'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
}
})
}
}
@@ -92,7 +92,7 @@ class ElasticUtils {
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));//goojia123456
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}