Merge branch 'script_gj' into 'master'

update

See merge request !721
This commit is contained in:
2023-10-09 18:02:31 +08:00
3 changed files with 227 additions and 45 deletions
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -499,8 +499,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2022-03-01"
def endDate = "2022-04-01"
def startDate = "2023-08-01"
def endDate = "2023-10-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -11,6 +11,7 @@ import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -36,7 +37,7 @@ import java.util.concurrent.atomic.AtomicLong
@BatchOwner(Watcher.GaoJuan)
class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static def version = new Date().getTime()
@@ -47,7 +48,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
@@ -59,11 +60,12 @@ class GaoDePoiDataProcess extends BaseBatch{
@Override
void process(Object args) {
// synchMapPoi();
// println ProvinceData.getProvince("成都")
synchMapPoi();
// synchCompany()
// poiCatProcess()
// addressAndCatProcess()
synchNetwork()
// synchNetwork()
}
def synchCompany(){
@@ -91,22 +93,26 @@ class GaoDePoiDataProcess extends BaseBatch{
def synchMapPoi(){
//"三明市","漳州市","厦门市","福州市"
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
def citys = ["福州市","重庆市"]
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys){
def townMap = findTownRange(city)
def results = CommonDatautils.findAmapDetailBySql(city)
println results.size()
def rows = running(results, "map")
save(city, rows)
println "city: "+city +" town: "+townMap.size()+" poi:"+results.size()
def rows = running(city, results, "map", townMap)
// save(city, rows)
saveToMG(city, rows)
}
}
def running(results, type, townMap){
def running(city,results, type, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def prov = ProvinceData.getProvince(city.replace("市",""))
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
@@ -114,7 +120,7 @@ class GaoDePoiDataProcess extends BaseBatch{
try{
totalCount.addAndGet(1)
if(type == "map"){
rets.addAll(createData(map, townMap))
rets.addAll(createData(prov, map, townMap))
}else if(type == "company"){
rets.addAll(createCompanyData(map, townMap))
}
@@ -209,15 +215,10 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def createData(map, townMap){
def fjw_province = null
if(map.city.contains("重庆")){
fjw_province = "重庆"
}else{
fjw_province = "福建"
}
def createData(province, map, townMap){
def fjw_province = province
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
map.id = new ObjectId().toString()
map.fjw_province = fjw_province
map.fjw_city = fjw_city
@@ -231,8 +232,14 @@ class GaoDePoiDataProcess extends BaseBatch{
if(arrs.size()>=3){
map.small_type = arrs[2]
}
if(arrs.size()>3){
map.cat = arrs[0]+";"+arrs[1]+";"+arrs[2]
}
}
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
@@ -251,18 +258,24 @@ class GaoDePoiDataProcess extends BaseBatch{
map.gps_location = gpslocation
}
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
if(!map.township){
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
}
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
}
}
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_district = fjw_region
}
map.fjw_district = fjw_region
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
@@ -320,6 +333,27 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def saveToMG(city, rows){
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def obj = new BasicDBObject()
m.each{k,v->
obj.put(k,v)
}
rets << obj
if(i%50000==0){
MAPPOI_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
MAPPOI_CONN.insert(rets)
rets.clear()
}
}
def getTownshipInfo(def glocation){
def township = null
def district = null
@@ -1,6 +1,9 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
@@ -14,31 +17,42 @@ import java.util.concurrent.atomic.AtomicLong
class PoiWash {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_test", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static def townsMap = [:]
public static def regions = []
static main(args) {
def task = new PoiWash()
task.run()
// task.check()
// def s = "002乡道与767县道交叉口东北约60米 "
// def s1 = DataUtil.toDBCcase(s)
// println s1
}
def run(){
//"三明市",
def citys = ["宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
def citys = ["泉州市", "三明市","宁德市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
for(def city : citys){
townsMap = findAllTownship(city)
regions = findAllRegions(city)
def datas = findPois(city)
println city +' '+datas.size()
println city +' poi: '+datas.size()+" towns:"+ townsMap.get(RegionData.formatCity(city)).size()
process(city, datas)
// updateOldRegion(city)
println city +" 完成"
}
}
def process(city, results){
// def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
@@ -70,7 +84,6 @@ class PoiWash {
}
latch.await()
// return rets;
}
@@ -85,10 +98,17 @@ class PoiWash {
addr = addr.replace(data.city,'')
}
if(data.district){
addr = addr.replace(data.district,'')
if(!addr.contains(data.district+"路")){
addr = addr.replace(data.district,'')
}
}
if(data.township){
addr = addr.replace(data.township,'')
try{
addr = addr.replace(data.township,'')
}catch(Exception e){
println data.id+" "+data.township
}
}
if(addr && addr.startsWith('-')){
addr = addr.replace('---','').replace('--','').replace('-','')
@@ -97,6 +117,12 @@ class PoiWash {
return null;
}
if(addr){
for(def r : regions){
addr = addr.replace(r, "")
}
}
if(addr){
def towns = townsMap.get(data.fjw_city)
for(def t : towns){
@@ -115,9 +141,11 @@ class PoiWash {
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("name", '三明海大饲料有限公司')
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
@@ -125,7 +153,7 @@ class PoiWash {
keys.put("address", 1)
keys.put("fjw_city", 1)
def dbcursor = MAPPOI_CONN.find(q, keys)
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
def rets = []
for(def d : dbcursor){
rets << d
@@ -133,13 +161,78 @@ class PoiWash {
return rets;
}
def updateOldRegion(def city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("fjw_city", 1)
keys.put("fjw_district", 1)
keys.put("township", 1)
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
def rets = []
def distMap = [:]
for(def d : dbcursor){
// def region = distMap.get(d.fjw_district)
// if(!region){
// distMap.put(d.fjw_district, d.district)
// }
//
// if(d.district instanceof Double){
// def newRegion = distMap.get(d.fjw_district)
// if(newRegion){
// println d.id +" "+d.fjw_district+" "+newRegion
// def udata = new BasicDBObject()
// udata.put("district", newRegion)
// MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
// }
// }
if(d.township instanceof Double){
def udata = new BasicDBObject()
udata.put("township", "")
MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
println d.id
}
}
return rets;
}
def findAllRegions(cityName){
def q = new BasicDBObject()
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
q.put("province_name", cityName)
}else{
q.put("city_name", cityName)
}
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
def set = new HashSet()
for(def d : dbcursor){
set << d.area_name
}
return set
}
def findAllTownship(cityName){
def q = new BasicDBObject()
q.put("city_name", cityName)
def dbcursor = TOWNSHIP_CONN.find(q)
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
q.put("province_name", cityName)
}else{
q.put("city_name", cityName)
}
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
def map = [:]
for(def d : dbcursor){
def city = d.fangjia_city
if(!city || city instanceof Double){
city = RegionData.formatCity(d.city_name)
}
def towns = map.get(city)
if(!towns){
towns = []
@@ -150,9 +243,64 @@ class PoiWash {
if(alias){
towns.addAll(alias)
}
towns.unique()
// towns.unique()
}
return map
}
def check(){
def citys = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市","三明市"]
for(def city : citys){
def datas = findPois(city)
processUpdate(city, datas)
println city +" 完成"
}
}
def processUpdate(city, results){
// def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(map.address){
def newAddr = DataUtil.toDBCcase(map.address)
if(newAddr.startsWith(";")){
newAddr = newAddr.substring(1)
}
if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
newAddr = ""
}
if(map.address!=newAddr){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", newAddr)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
// return rets;
}
}