Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy
This commit is contained in:
2024-08-23 11:15:00 +08:00
917 changed files with 36385 additions and 12401 deletions
@@ -24,7 +24,7 @@ class AddThirdPriceSample {
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
@@ -37,9 +37,9 @@ class MinglujiData {
// matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
saveMljData()
fiveLevelSplit(city)
// fiveLevelSplit(city)
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
@@ -227,7 +227,7 @@ class MinglujiData {
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -32,6 +32,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -126,7 +127,7 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ '绵阳市', '聊城市', '肇庆市', '自贡市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
def buducitys = [ '茂名市', '荆州市', '荆门市', '莆田市','苏州市',
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
@@ -167,7 +168,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def runbuducitys = ["湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",
'绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市',
'东营市', '中卫市']
'东营市', '中卫市', '绵阳市', '聊城市', '肇庆市', '自贡市', '芜湖市']
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
@@ -184,8 +185,8 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def keyRets = new CopyOnWriteArrayList()
def rets = new ConcurrentHashMap()
//def keyRets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
// def townMap = findTownRange(city)
@@ -204,9 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
if(ret && !keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
keyRets << ret.city+"##"+ret.district+"##"+ret.name
rets << ret
if(ret && !rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){
//keyRets << ret.city+"##"+ret.district+"##"+ret.name
//rets << ret
rets.put(ret.city+"##"+ret.district+"##"+ret.name, ret)
}
}
@@ -698,8 +700,8 @@ class GaoDePoiDataProcess extends BaseBatch{
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
rows.each{key,map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."