suggestion 增加判断已经处理过的城市

This commit is contained in:
mjt
2024-01-29 15:34:21 +08:00
parent 7447715612
commit 6c769c15ff
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.lixue.DB.City
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
@@ -20,13 +21,15 @@ import java.util.concurrent.atomic.AtomicLong
class SuggestionDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -42,14 +45,31 @@ class SuggestionDataSynch extends BaseBatch{
static main(args) {
runBatch(new SuggestionDataSynch(), args);
}
@Override
public void process(Object args) {
run()
// updateCode()
// getExitsCity()
}
def getExitsCity(){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
def cities = SUGGESTION_CONN.distinct("city")
def simpleCities = []
for (def c :cities) {
def city = RegionData.formatCity(c)
simpleCities.add(city)
}
// print(simpleCities)
return simpleCities
}
def run() {
@@ -58,22 +78,74 @@ class SuggestionDataSynch extends BaseBatch{
def index = 0
fullCityMap = queryFullNameCity()
mcaDistMap = queryDistNameCity()
def existCities = getExitsCity()
for (def c : citys) {
def allRets = []
// if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){
if(existCities.contains(c)){
continue;
}
index++
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
updating(c, "district")
updating(c, "office")
updating(c, "shop")
println index + " " + c + " 完成"
// }
if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
poiData(c)
}
def ret1 = updating(c, "district")
def ret2 = updating(c, "office")
def ret3 = updating(c, "shop")
// }
// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
def ret4 = poiData(c)
// }
allRets.addAll(ret1)
allRets.addAll(ret2)
allRets.addAll(ret3)
allRets.addAll(ret4)
saveToMG(c, allRets)
println index + " " + c + " 完成 "+ allRets.size()
}
}
def saveToMG(city, rows){
def data = fullCityMap.get(city)
def q = new BasicDBObject("city", data.city)
def count = 0
try{
count = SUGGESTION_CONN.count(q)
}catch(e){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
count = SUGGESTION_CONN.count(q)
}
// if(count>0){
// SUGGESTION_CONN.remove(q)
// }
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def obj = new BasicDBObject()
m.each{k,v->
obj.put(k,v)
}
rets << obj
if(i%50000==0){
SUGGESTION_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
SUGGESTION_CONN.insert(rets)
rets.clear()
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
def rets = new CopyOnWriteArrayList()
@@ -93,7 +165,8 @@ class SuggestionDataSynch extends BaseBatch{
def map = createData(dist)
//往100 上同步
updateData(map)
// updateData(map)
rets << map
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
@@ -135,7 +208,7 @@ class SuggestionDataSynch extends BaseBatch{
map.alias = dist.alias
}
if(dist.addr_alias && dist.addr_alias.size()>0){
map.addr_alias = dist.addr_alias
map.addr_alias = dist.addr_alias
}
map.type = dist.cat
if(dist.lng){
@@ -178,17 +251,19 @@ class SuggestionDataSynch extends BaseBatch{
def data = fullCityMap.get(city)
if(data){
def results = CommonDatautils.findAmapDetailBySql(data.city)
createPoi(results, data, townMap)
println city +" 完成!"
def rets = createPoi(data.city, results, data, townMap)
return rets;
}
return []
}
def createPoi(results, data, townMap){
def createPoi(fullCity, results, data, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
def townshipMap = findMapPoiTownshipMap(fullCity)
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
final def dist = results[i]
@@ -213,16 +288,25 @@ class SuggestionDataSynch extends BaseBatch{
map.location = dist.location
map.source = "FJW"
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
if(townshipMap && townshipMap.containsKey(dist.id)){
def township = townshipMap.get(dist.id)
if(township){
map.township = township
}
}
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
}
}
println map.township
//往100 上同步
updateData(map)
// updateData(map)
rets << map
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
@@ -238,6 +322,30 @@ class SuggestionDataSynch extends BaseBatch{
return rets;
}
def findMapPoiTownshipMap(fullCity){
def q = new BasicDBObject()
q.put("city", fullCity)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("township", 1)
def dbcursor = MAPPOI_CONN.find(q, keys)
def map = [:]
for(def d : dbcursor){
map.put(d.id, d.township)
}
return map;
}
def findTownshipByBD(id){
def q = new BasicDBObject()
q.put("id", id)
def obj = MAPPOI_CONN.findOne(q)
return obj
}
def queryFullNameCity(){
def q = new BasicDBObject()
@@ -251,13 +359,13 @@ class SuggestionDataSynch extends BaseBatch{
if(d.city == "市辖区"){
fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
d.city = d.province
d.city = d.province
}
}
}
}else if(d.level == "区域"){
if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.city)
fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
d.city = d.district
}
@@ -285,7 +393,7 @@ class SuggestionDataSynch extends BaseBatch{
}else if(d.city == "市辖区"){
def fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}else{