Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-03-19 09:11:19 +08:00
2 changed files with 182 additions and 0 deletions
@@ -7,6 +7,7 @@ import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.newDistrict.InvokeHttpRequest
import com.fangjia.script.util.*
@@ -5618,4 +5619,19 @@ class CommonDatautils {
return bk_business_coll.find(q).toList()
}
static def findOfficeByMap(city, small_type){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("small_type", small_type))
def rets = []
def index= 0
ElasticUtils89.executeQuery("192.168.0.89", "gaode_map_poi_2024", queryBuilder, { String id, Map map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.data_id,"province":map.province,"city":map.city, "district":map.district, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.lon, "lat":map.location.lat, "parent_id":map.parent]
// }
})
return rets;
}
}
@@ -0,0 +1,166 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AutoAddOfficeBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
runBatch(new AutoAddOfficeBatch(), args)
}
@Override
void process(Object args) {
// def pattern = /[A-Z]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|/
// println "开放广场商务写字楼C栋".replaceAll(pattern, "")
def citys = ["临沂"]//CommonDatautils.findAllCity()
for(def c : citys){
def cityObj = fullCityMap.get(c)
if(cityObj){
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
officeMaps = filterOffice(officeMaps)
def offices = matchOffice(officeMaps)
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", offices, "数据")
}
}
}
def filterOffice(officeMaps){
def offices = []
def keys = []
for(def o : officeMaps){
println o.map_id
if(o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.contains("综合办公楼")){
println "[ 过滤 ] "+o.city+" "+o.name
continue
}
if(o.name.contains("路写字楼")){
println "[ 过滤 ] "+o.city+" "+o.name
continue
}
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|.|[A-Z]楼|[0-9]幢/
o.name = o.name.replaceAll(pattern, "")
def keyName = o.city +"#"+o.district+"#"+o.name
if(!keys.contains(keyName)){
offices << o
keys << keyName
}
}
println "原:"+officeMaps.size() +" 新:"+offices.size()
return offices
}
def matchOffice(offices){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
rets << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
for(def i=0;i<offices.size();i++){
final def nmd = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def office = matchDB(nmd)
if(!office){
office = matchApi(nmd)
}
if(office){
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
println addCount.get()+"/"+total +" [success] "+ office.get("region")+" "+office.get("name")+" "+office.match_way
}else{
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address,nmd.lng, nmd.lat,"","","",""]
println addCount.get()+"/"+total +" [fail] "+nmd.city+" "+nmd.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def matchDB(nmd){
def simpleCity = RegionData.formatCity(nmd.city)
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
if(simpleCity && simpleDistrict){
def office = null
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
if(rets){
if(rets.size()>=1){
office = rets[0]
}
}else{
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
if(rets && rets.size()>=1){
office = rets[0]
}else{
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
if(rets && rets.size()>=1){
office = rets[0]
}
}
}
if(office){
office.match_way = "数据库"
}
return office
}
return null
}
def matchApi(nmd){
def address = nmd.city
if(nmd.district){
address = address + nmd.district
}
address = address + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
if(!jsonObject){
if(DataUtil.isPreciseAddress(nmd.address)){
address = nmd.city
if(nmd.district){
address = address + nmd.district
}
address = address + nmd.address
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
}
}
if(jsonObject) {
def udata = [:]
udata.put("_id",jsonObject.get("id"))
udata.put("city",jsonObject.get("city"))
udata.put("region",jsonObject.get("district"))
udata.put("name",jsonObject.get("name"))
udata.put("match_type",jsonObject.get("propertyType"))
udata.put("match_credit",jsonObject.get("credit"))
udata.put("match_way", "接口")
return udata
}
return null
}
}