Merge branch 'script_gj' into 'master'

Script gj

See merge request !926
This commit is contained in:
2024-08-23 11:16:37 +08:00
7 changed files with 21 additions and 14 deletions
@@ -24,7 +24,7 @@ class AddThirdPriceSample {
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
@@ -4746,7 +4746,7 @@ class CommonDatautils {
townMap.put(key, list)
}
println "乡镇街道 加载完成"
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
@@ -15,13 +15,16 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
}
if(suggestName){
url = url +"&suggestName="+suggestName
}
if(suggestDistrict){
url = url +"&suggestDistrict="+suggestDistrict
}
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
@@ -37,9 +37,9 @@ class MinglujiData {
// matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
saveMljData()
fiveLevelSplit(city)
// fiveLevelSplit(city)
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
@@ -227,7 +227,7 @@ class MinglujiData {
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
@@ -335,7 +335,7 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -127,7 +127,7 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ '绵阳市', '聊城市', '肇庆市', '自贡市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
def buducitys = [ '茂名市', '荆州市', '荆门市', '莆田市','苏州市',
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
@@ -168,7 +168,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def runbuducitys = ["湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",
'绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市',
'东营市', '中卫市']
'东营市', '中卫市', '绵阳市', '聊城市', '肇庆市', '自贡市', '芜湖市']
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
@@ -205,7 +205,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
if(!rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){
if(ret && !rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){
//keyRets << ret.city+"##"+ret.district+"##"+ret.name
//rets << ret
rets.put(ret.city+"##"+ret.district+"##"+ret.name, ret)
@@ -319,6 +319,11 @@ class GaoDePoiDataProcess extends BaseBatch{
// retMap.small_type = "村庄级地名"
retMap.cat = map.std_tag
if(!map.std_tag){
println map.name +" std_tag: "+ map.std_tag +" 为空过滤"
return null
}
def nameArr = map.std_tag.split(";|:")
if(nameArr.size()>=1){
retMap.type = nameArr[0]
@@ -695,8 +700,8 @@ class GaoDePoiDataProcess extends BaseBatch{
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
for (String key : rows.keySet()) {
def map = rows.get(key)
rows.each{key,map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
@@ -730,7 +735,6 @@ class GaoDePoiDataProcess extends BaseBatch{
}
}
println "[${city}] 结束时间 "+new Date()
}