Merge branch 'dev' of http://gitlab.fangjia.com/dev0/main into dev
This commit is contained in:
+130
-22
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
|
|||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.lixue.DB.City
|
||||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
@@ -20,13 +21,15 @@ import java.util.concurrent.atomic.AtomicLong
|
|||||||
class SuggestionDataSynch extends BaseBatch{
|
class SuggestionDataSynch extends BaseBatch{
|
||||||
|
|
||||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||||
|
|
||||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||||
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||||
|
|
||||||
|
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
|
||||||
|
|
||||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||||
|
|
||||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
@@ -42,14 +45,31 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new SuggestionDataSynch(), args);
|
runBatch(new SuggestionDataSynch(), args);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
run()
|
run()
|
||||||
// updateCode()
|
// updateCode()
|
||||||
|
// getExitsCity()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getExitsCity(){
|
||||||
|
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||||
|
def cities = SUGGESTION_CONN.distinct("city")
|
||||||
|
def simpleCities = []
|
||||||
|
for (def c :cities) {
|
||||||
|
def city = RegionData.formatCity(c)
|
||||||
|
simpleCities.add(city)
|
||||||
|
}
|
||||||
|
// print(simpleCities)
|
||||||
|
return simpleCities
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def run() {
|
def run() {
|
||||||
@@ -58,22 +78,74 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
def index = 0
|
def index = 0
|
||||||
fullCityMap = queryFullNameCity()
|
fullCityMap = queryFullNameCity()
|
||||||
mcaDistMap = queryDistNameCity()
|
mcaDistMap = queryDistNameCity()
|
||||||
|
def existCities = getExitsCity()
|
||||||
|
|
||||||
for (def c : citys) {
|
for (def c : citys) {
|
||||||
|
|
||||||
|
def allRets = []
|
||||||
|
|
||||||
|
// if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){
|
||||||
|
if(existCities.contains(c)){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
index++
|
index++
|
||||||
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||||
updating(c, "district")
|
def ret1 = updating(c, "district")
|
||||||
updating(c, "office")
|
def ret2 = updating(c, "office")
|
||||||
updating(c, "shop")
|
def ret3 = updating(c, "shop")
|
||||||
println index + " " + c + " 完成"
|
|
||||||
// }
|
|
||||||
if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
|
|
||||||
poiData(c)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
// }
|
||||||
|
// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
|
||||||
|
def ret4 = poiData(c)
|
||||||
|
// }
|
||||||
|
|
||||||
|
allRets.addAll(ret1)
|
||||||
|
allRets.addAll(ret2)
|
||||||
|
allRets.addAll(ret3)
|
||||||
|
allRets.addAll(ret4)
|
||||||
|
|
||||||
|
saveToMG(c, allRets)
|
||||||
|
println index + " " + c + " 完成 "+ allRets.size()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def saveToMG(city, rows){
|
||||||
|
|
||||||
|
def data = fullCityMap.get(city)
|
||||||
|
def q = new BasicDBObject("city", data.city)
|
||||||
|
def count = 0
|
||||||
|
try{
|
||||||
|
count = SUGGESTION_CONN.count(q)
|
||||||
|
}catch(e){
|
||||||
|
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||||
|
count = SUGGESTION_CONN.count(q)
|
||||||
|
}
|
||||||
|
// if(count>0){
|
||||||
|
// SUGGESTION_CONN.remove(q)
|
||||||
|
// }
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
for(def i=0;i<rows.size();i++){
|
||||||
|
def m = rows[i]
|
||||||
|
def obj = new BasicDBObject()
|
||||||
|
m.each{k,v->
|
||||||
|
obj.put(k,v)
|
||||||
|
}
|
||||||
|
rets << obj
|
||||||
|
if(i%50000==0){
|
||||||
|
SUGGESTION_CONN.insert(rets)
|
||||||
|
rets.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(rets.size()>0){
|
||||||
|
SUGGESTION_CONN.insert(rets)
|
||||||
|
rets.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def updating(city, cat){
|
def updating(city, cat){
|
||||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
|
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
|
||||||
def rets = new CopyOnWriteArrayList()
|
def rets = new CopyOnWriteArrayList()
|
||||||
@@ -93,7 +165,8 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
def map = createData(dist)
|
def map = createData(dist)
|
||||||
//往100 上同步
|
//往100 上同步
|
||||||
updateData(map)
|
// updateData(map)
|
||||||
|
rets << map
|
||||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -135,7 +208,7 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
map.alias = dist.alias
|
map.alias = dist.alias
|
||||||
}
|
}
|
||||||
if(dist.addr_alias && dist.addr_alias.size()>0){
|
if(dist.addr_alias && dist.addr_alias.size()>0){
|
||||||
map.addr_alias = dist.addr_alias
|
map.addr_alias = dist.addr_alias
|
||||||
}
|
}
|
||||||
map.type = dist.cat
|
map.type = dist.cat
|
||||||
if(dist.lng){
|
if(dist.lng){
|
||||||
@@ -178,17 +251,19 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
def data = fullCityMap.get(city)
|
def data = fullCityMap.get(city)
|
||||||
if(data){
|
if(data){
|
||||||
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
||||||
createPoi(results, data, townMap)
|
def rets = createPoi(data.city, results, data, townMap)
|
||||||
println city +" 完成!"
|
return rets;
|
||||||
}
|
}
|
||||||
|
return []
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def createPoi(results, data, townMap){
|
def createPoi(fullCity, results, data, townMap){
|
||||||
def rets = new CopyOnWriteArrayList()
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
def totalCount = new AtomicLong()
|
def totalCount = new AtomicLong()
|
||||||
|
def townshipMap = findMapPoiTownshipMap(fullCity)
|
||||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||||
for(def i=0;i<results.size();i++){
|
for(def i=0;i<results.size();i++){
|
||||||
final def dist = results[i]
|
final def dist = results[i]
|
||||||
@@ -213,16 +288,25 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
map.location = dist.location
|
map.location = dist.location
|
||||||
map.source = "FJW"
|
map.source = "FJW"
|
||||||
|
|
||||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
if(townshipMap && townshipMap.containsKey(dist.id)){
|
||||||
|
def township = townshipMap.get(dist.id)
|
||||||
|
if(township){
|
||||||
|
map.township = township
|
||||||
|
}
|
||||||
|
}
|
||||||
if(!map.township){
|
if(!map.township){
|
||||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||||
if(gaodeMap && gaodeMap.township){
|
if(!map.township){
|
||||||
map.township = gaodeMap.township
|
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||||
|
if(gaodeMap && gaodeMap.township){
|
||||||
|
map.township = gaodeMap.township
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println map.township
|
println map.township
|
||||||
//往100 上同步
|
//往100 上同步
|
||||||
updateData(map)
|
// updateData(map)
|
||||||
|
rets << map
|
||||||
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -238,6 +322,30 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
return rets;
|
return rets;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def findMapPoiTownshipMap(fullCity){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", fullCity)
|
||||||
|
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("id", 1)
|
||||||
|
keys.put("township", 1)
|
||||||
|
def dbcursor = MAPPOI_CONN.find(q, keys)
|
||||||
|
def map = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
map.put(d.id, d.township)
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findTownshipByBD(id){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("id", id)
|
||||||
|
def obj = MAPPOI_CONN.findOne(q)
|
||||||
|
return obj
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def queryFullNameCity(){
|
def queryFullNameCity(){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
@@ -251,13 +359,13 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
if(d.city == "市辖区"){
|
if(d.city == "市辖区"){
|
||||||
fjwcity = RegionData.formatCity(d.province)
|
fjwcity = RegionData.formatCity(d.province)
|
||||||
if(fjwcity){
|
if(fjwcity){
|
||||||
d.city = d.province
|
d.city = d.province
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else if(d.level == "区域"){
|
}else if(d.level == "区域"){
|
||||||
if(d.administer == "省直辖县级行政区划"){
|
if(d.administer == "省直辖县级行政区划"){
|
||||||
fjwcity = RegionData.formatCity(d.city)
|
fjwcity = RegionData.formatCity(d.city)
|
||||||
if(fjwcity){
|
if(fjwcity){
|
||||||
d.city = d.district
|
d.city = d.district
|
||||||
}
|
}
|
||||||
@@ -285,7 +393,7 @@ class SuggestionDataSynch extends BaseBatch{
|
|||||||
}else if(d.city == "市辖区"){
|
}else if(d.city == "市辖区"){
|
||||||
def fjwcity = RegionData.formatCity(d.province)
|
def fjwcity = RegionData.formatCity(d.province)
|
||||||
if(fjwcity){
|
if(fjwcity){
|
||||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
|
|||||||
Reference in New Issue
Block a user