Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-08-23 11:15:30 +08:00
2 changed files with 57 additions and 40 deletions
@@ -31,15 +31,15 @@ class MinglujiData {
def run(){
def prov = "浙江省"
def city = "嘉兴"
def city = "湖州"
// exportDatas(city)
// matchDev(prov, city)
matchCommunity(city)
// matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
fiveLevelSplit(city)
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
@@ -48,11 +48,11 @@ class MinglujiData {
def matchDev(prov, city){
def devs = loadAllDevs(prov)
def i = 1;
def i = 2;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
@@ -152,11 +152,11 @@ class MinglujiData {
}
def matchCommunity(city){
def i = 1;
def i = 8;
while(i<20){
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
}
File file=new File(fileName);
@@ -203,6 +203,29 @@ class MinglujiData {
}
def fiveLevelSplit(city){
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file = new File(fileName);
if (file.exists()) {
ExcelUtils.multiThreadingModify(fileName, { d ->
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['E'])
if(jsonObject) {
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
d['J'] = jsonObject.get("address")
}
}
})
}
i++
}
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
def obj = new BasicDBObject()
@@ -213,7 +236,7 @@ class MinglujiData {
obj.put("address", d['E'])
obj.put("dev_name", d['F'])
obj.put("committee", d['G'])
obj.put("street", d['H'])
obj.put("township", d['H'])
company_100_CONN.save(obj)
})
}
@@ -32,6 +32,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -126,7 +127,10 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
def buducitys = [ '绵阳市', '聊城市', '肇庆市', '自贡市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
'丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市',
'云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市',
'佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州',
@@ -136,7 +140,7 @@ class GaoDePoiDataProcess extends BaseBatch{
'吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市',
'哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市',
'固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市',
'天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市',
'天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市',
'安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市',
'宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市',
'巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市',
@@ -149,17 +153,12 @@ class GaoDePoiDataProcess extends BaseBatch{
'梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市',
'汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市',
'泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市',
'淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市',
'淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市',
'湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市',
'潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市',
'珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县',
'白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市',
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市',
'肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市',
'邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市',
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市',
'重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市',
'锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市',
'阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市',
@@ -167,34 +166,35 @@ class GaoDePoiDataProcess extends BaseBatch{
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
//"丽水市","杭州市",
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市',
'东营市', '中卫市', '中山市', '临夏回族自治州']
def runbuducitys = ["湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",
'绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市',
'东营市', '中卫市']
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
}
def pois = findAllVillagePoiMap(city)
println "百度loading finish: "+pois.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss")
def rows = createVillageData(city, pois)
save(city, rows)
// saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
println city +" "+pois.size()+" add:"+rows.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def keyRets = new CopyOnWriteArrayList()
def rets = new ConcurrentHashMap()
//def keyRets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
// def townMap = findTownRange(city)
def townMap = CommonDatautils.queryTownByCity(city)
def poiKeysFilter = findEsPoiByCity(city)
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
for(def i=0;i<pois.size();i++){
def p = pois[i]
final def p = pois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
@@ -205,9 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
keyRets.addAll(ret)
rets << ret
if(!rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){
//keyRets << ret.city+"##"+ret.district+"##"+ret.name
//rets << ret
rets.put(ret.city+"##"+ret.district+"##"+ret.name, ret)
}
}
@@ -225,16 +226,9 @@ class GaoDePoiDataProcess extends BaseBatch{
def findEsPoiByCity(city){
// def sql = "select city,district,name,location from didata_map_poi_single where city='${city}' "
// def rets = new HashSet()
// ElasticUtils89.executeSqlQuery(sql, { d->
//
// })
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
// def rets = new HashSet()
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
// rets << d.city +"##"+d.district+"##"+d.name
filter.put(d.city +"##"+d.district+"##"+d.name)
})
return filter
@@ -429,7 +423,7 @@ class GaoDePoiDataProcess extends BaseBatch{
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
keys.put("std_tag", 1)
def list = BAIDU_POI_COLL.find(q, keys).toList()
def list = BAIDU_POI_COLL.find(q, keys).batchSize(2000).toList()
return list;
}
@@ -701,8 +695,8 @@ class GaoDePoiDataProcess extends BaseBatch{
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
for (String key : rows.keySet()) {
def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
@@ -717,7 +711,7 @@ class GaoDePoiDataProcess extends BaseBatch{
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();