Merge branch 'script_gj' into 'master'

poi地址拆分

See merge request !744
This commit is contained in:
2023-10-31 14:49:41 +08:00
2 changed files with 116 additions and 2 deletions
@@ -114,6 +114,27 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def structure(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -225,6 +246,32 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def newMatchPrice(city,address, property, useMap){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://localhost:8080/fangjia-open/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
if(property){
url = url + "&category=${property}"
}
url = url + "&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newStandard(address){
def jsonObj = null
def content = null
@@ -28,7 +28,7 @@ class PoiWash {
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static def townsMap = [:]
public static def regions = []
@@ -41,8 +41,9 @@ class PoiWash {
// task.synch()
// task.check()
// task.deduplicate()
task.updateEsMapPoi()
// task.updateEsMapPoi()
// task.checkSpecialSymbols()
task.splitRoad()
}
def updateId(){
@@ -449,6 +450,54 @@ class PoiWash {
}
}
def splitRoad(){
//"三明市",
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
for(def city : citys) {
def datas = findPoiByQuery(city, ['city','address'])
splitRoadRun(datas)
println city +" 完成"
}
}
def splitRoadRun(results){
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
if(jsonObject && jsonObject.containsKey("result")){
def dataMap = jsonObject.get("result")
if(dataMap.get("street")){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("street", dataMap.get("street"))
if(dataMap.get("streetNumber")){
udata.put("street_number", dataMap.get("streetNumber"))
}
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def updateMapPoi(def id){
def header = [:]
@@ -494,6 +543,24 @@ class PoiWash {
}
def findPoiByQuery(city, fields){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
fields.each{
keys.put(it, 1)
}
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(5000)
def rets = []
for (def d : dbcursor) {
rets << d
}
return rets;
}
def check(){
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
def citys = ["重庆市"]