Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+16
@@ -7,6 +7,7 @@ import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.newDistrict.InvokeHttpRequest
|
||||
import com.fangjia.script.util.*
|
||||
@@ -5618,4 +5619,19 @@ class CommonDatautils {
|
||||
return bk_business_coll.find(q).toList()
|
||||
}
|
||||
|
||||
|
||||
static def findOfficeByMap(city, small_type){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("small_type", small_type))
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "gaode_map_poi_2024", queryBuilder, { String id, Map map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
|
||||
index++
|
||||
rets << ["map_id":map.data_id,"province":map.province,"city":map.city, "district":map.district, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.lon, "lat":map.location.lat, "parent_id":map.parent]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AutoAddOfficeBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AutoAddOfficeBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
// def pattern = /[A-Z]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|/
|
||||
// println "开放广场商务写字楼C栋".replaceAll(pattern, "")
|
||||
|
||||
def citys = ["临沂"]//CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def cityObj = fullCityMap.get(c)
|
||||
if(cityObj){
|
||||
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
|
||||
officeMaps = filterOffice(officeMaps)
|
||||
def offices = matchOffice(officeMaps)
|
||||
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", offices, "数据")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def filterOffice(officeMaps){
|
||||
def offices = []
|
||||
def keys = []
|
||||
for(def o : officeMaps){
|
||||
println o.map_id
|
||||
if(o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.contains("综合办公楼")){
|
||||
println "[ 过滤 ] "+o.city+" "+o.name
|
||||
continue
|
||||
}
|
||||
if(o.name.contains("路写字楼")){
|
||||
println "[ 过滤 ] "+o.city+" "+o.name
|
||||
continue
|
||||
}
|
||||
|
||||
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|.|[A-Z]楼|[0-9]幢/
|
||||
o.name = o.name.replaceAll(pattern, "")
|
||||
|
||||
def keyName = o.city +"#"+o.district+"#"+o.name
|
||||
if(!keys.contains(keyName)){
|
||||
offices << o
|
||||
keys << keyName
|
||||
}
|
||||
}
|
||||
println "原:"+officeMaps.size() +" 新:"+offices.size()
|
||||
return offices
|
||||
}
|
||||
|
||||
def matchOffice(offices){
|
||||
def totalCount = new AtomicLong()
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
def total = offices.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
rets << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
|
||||
for(def i=0;i<offices.size();i++){
|
||||
final def nmd = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
|
||||
def office = matchDB(nmd)
|
||||
if(!office){
|
||||
office = matchApi(nmd)
|
||||
}
|
||||
if(office){
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
|
||||
println addCount.get()+"/"+total +" [success] "+ office.get("region")+" "+office.get("name")+" "+office.match_way
|
||||
}else{
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address,nmd.lng, nmd.lat,"","","",""]
|
||||
println addCount.get()+"/"+total +" [fail] "+nmd.city+" "+nmd.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def matchDB(nmd){
|
||||
def simpleCity = RegionData.formatCity(nmd.city)
|
||||
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
|
||||
if(simpleCity && simpleDistrict){
|
||||
def office = null
|
||||
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
|
||||
if(rets){
|
||||
if(rets.size()>=1){
|
||||
office = rets[0]
|
||||
}
|
||||
}else{
|
||||
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
|
||||
if(rets && rets.size()>=1){
|
||||
office = rets[0]
|
||||
}else{
|
||||
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
|
||||
if(rets && rets.size()>=1){
|
||||
office = rets[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if(office){
|
||||
office.match_way = "数据库"
|
||||
}
|
||||
return office
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def matchApi(nmd){
|
||||
def address = nmd.city
|
||||
if(nmd.district){
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.name
|
||||
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
|
||||
if(!jsonObject){
|
||||
if(DataUtil.isPreciseAddress(nmd.address)){
|
||||
address = nmd.city
|
||||
if(nmd.district){
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.address
|
||||
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
|
||||
}
|
||||
}
|
||||
if(jsonObject) {
|
||||
def udata = [:]
|
||||
udata.put("_id",jsonObject.get("id"))
|
||||
udata.put("city",jsonObject.get("city"))
|
||||
udata.put("region",jsonObject.get("district"))
|
||||
udata.put("name",jsonObject.get("name"))
|
||||
udata.put("match_type",jsonObject.get("propertyType"))
|
||||
udata.put("match_credit",jsonObject.get("credit"))
|
||||
udata.put("match_way", "接口")
|
||||
return udata
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user