suggestion修改同步

This commit is contained in:
2024-01-08 18:08:48 +08:00
parent 7502825b1b
commit 44a0c654d6
3 changed files with 174 additions and 25 deletions
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -18,25 +19,24 @@ class EsSearchUtils {
static def findPois(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
def rets = []
while (rs.next()) {
def m = [:]
m.city = rs.getString("cityname")
m.region = rs.getString("adname")
m.name = rs.getString("name")
m.address = rs.getString("address")
m.type = rs.getString("type")
m.code = rs.getString("typecode")
m.lng = rs.getString("location.lon")
m.lat = rs.getString("location.lat")
rets << m
}
ElasticUtils.executeSqlQuery(sql,{ data->
def m = [:]
m.city = data.cityname
m.region = data.adname
m.name = data.name
m.address = data.address
m.type = data.type
m.code = data.typecode
def location = data.location
m.lng = location.lon
m.lat = location.lat
rets << m
})
return rets;
}
@@ -0,0 +1,110 @@
package com.fangjia.script.gj.village
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def filePath = "D:\\数据\\涉农\\县.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
// if(!row['D']){
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
// if(rets){
// def obj = findOneByName(rets, row['C']+"人民政府")
// if(obj){
// row['D'] = obj.lng
// row['E'] = obj.lat
// println row['C'] + " "+obj
// }
// }
// }
// if(row['D'] && !row['F']){
// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
// def map = ["lon":row['D'], "lat":row['E']]
// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
// row['F'] = township
// }
if(row['F'] && !row['K']){
def q = new BasicDBObject()
q.put("city_name", row['B'])
q.put("area_name", row['C'])
def streetName = row['I']
q.put("street_name", streetName)
def count = COMMUNITY_CONN.count(q);
row['K'] = count
// row['H'] = streetName
}
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
// if(gaodeMap && gaodeMap.township){
// row['I'] = gaodeMap.township
// }
})
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
def findOneByName(def rets, name){
def obj = null
for(def r : rets){
if(r.name == name){
obj = r
break;
}
}
return obj
}
def findTownRange(city, TOWN_POLYLINECOLL){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
@@ -63,22 +63,59 @@ class SuggestionDataSynch extends BaseBatch{
mcaDistMap = queryDistNameCity()
for (def c : citys) {
def allRets = []
index++
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
updating(c, "district")
updating(c, "office")
updating(c, "shop")
def ret1 = updating(c, "district")
def ret2 = updating(c, "office")
def ret3 = updating(c, "shop")
// }
// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
poiData(c)
def ret4 = poiData(c)
// }
println index + " " + c + " 完成"
allRets.addAll(ret1)
allRets.addAll(ret2)
allRets.addAll(ret3)
allRets.addAll(ret4)
println index + " " + c + " 完成"
saveToMG(c, allRets)
}
}
def saveToMG(city, rows){
def data = fullCityMap.get(city)
def q = new BasicDBObject("city", data.city)
def count = SUGGESTION_CONN.count(q)
if(count>0){
SUGGESTION_CONN.remove(q)
}
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def obj = new BasicDBObject()
m.each{k,v->
obj.put(k,v)
}
rets << obj
if(i%50000==0){
SUGGESTION_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
SUGGESTION_CONN.insert(rets)
rets.clear()
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
def rets = new CopyOnWriteArrayList()
@@ -98,7 +135,8 @@ class SuggestionDataSynch extends BaseBatch{
def map = createData(dist)
//往100 上同步
updateData(map)
// updateData(map)
rets << map
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
@@ -227,7 +265,8 @@ class SuggestionDataSynch extends BaseBatch{
}
println map.township
//往100 上同步
updateData(map)
// updateData(map)
rets << map
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {