+4
-4
@@ -40,8 +40,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -499,8 +499,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
def startDate = "2023-08-01"
|
||||
def endDate = "2023-10-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
|
||||
+62
-28
@@ -11,6 +11,7 @@ import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -36,7 +37,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
|
||||
@@ -47,7 +48,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
|
||||
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
@@ -59,11 +60,12 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// synchMapPoi();
|
||||
// println ProvinceData.getProvince("成都")
|
||||
synchMapPoi();
|
||||
// synchCompany()
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
synchNetwork()
|
||||
// synchNetwork()
|
||||
}
|
||||
|
||||
def synchCompany(){
|
||||
@@ -91,22 +93,26 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def synchMapPoi(){
|
||||
//"三明市","漳州市","厦门市","福州市"
|
||||
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
|
||||
def citys = ["福州市","重庆市"]
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys){
|
||||
def townMap = findTownRange(city)
|
||||
def results = CommonDatautils.findAmapDetailBySql(city)
|
||||
println results.size()
|
||||
def rows = running(results, "map")
|
||||
save(city, rows)
|
||||
println "city: "+city +" town: "+townMap.size()+" poi:"+results.size()
|
||||
def rows = running(city, results, "map", townMap)
|
||||
// save(city, rows)
|
||||
saveToMG(city, rows)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def running(results, type, townMap){
|
||||
def running(city,results, type, townMap){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def prov = ProvinceData.getProvince(city.replace("市",""))
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -114,7 +120,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(type == "map"){
|
||||
rets.addAll(createData(map, townMap))
|
||||
rets.addAll(createData(prov, map, townMap))
|
||||
}else if(type == "company"){
|
||||
rets.addAll(createCompanyData(map, townMap))
|
||||
}
|
||||
@@ -209,15 +215,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def createData(map, townMap){
|
||||
def fjw_province = null
|
||||
if(map.city.contains("重庆")){
|
||||
fjw_province = "重庆"
|
||||
}else{
|
||||
fjw_province = "福建"
|
||||
}
|
||||
def createData(province, map, townMap){
|
||||
def fjw_province = province
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
map.id = new ObjectId().toString()
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
@@ -231,8 +232,14 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
if(arrs.size()>=3){
|
||||
map.small_type = arrs[2]
|
||||
}
|
||||
if(arrs.size()>3){
|
||||
map.cat = arrs[0]+";"+arrs[1]+";"+arrs[2]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
@@ -251,18 +258,24 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
if(!map.township){
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
}
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
|
||||
map.fjw_district = fjw_region
|
||||
}
|
||||
map.fjw_district = fjw_region
|
||||
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
@@ -320,6 +333,27 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def saveToMG(city, rows){
|
||||
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def obj = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
rets << obj
|
||||
if(i%50000==0){
|
||||
MAPPOI_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
MAPPOI_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
|
||||
+161
-13
@@ -1,6 +1,9 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
@@ -14,31 +17,42 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PoiWash {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_test", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static def townsMap = [:]
|
||||
public static def regions = []
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
task.run()
|
||||
// task.check()
|
||||
|
||||
// def s = "002乡道与767县道交叉口东北约60米 "
|
||||
// def s1 = DataUtil.toDBCcase(s)
|
||||
// println s1
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
//"三明市",
|
||||
def citys = ["宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
|
||||
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
|
||||
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
|
||||
def citys = ["泉州市", "三明市","宁德市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
|
||||
for(def city : citys){
|
||||
townsMap = findAllTownship(city)
|
||||
regions = findAllRegions(city)
|
||||
def datas = findPois(city)
|
||||
println city +' '+datas.size()
|
||||
println city +' poi: '+datas.size()+" towns:"+ townsMap.get(RegionData.formatCity(city)).size()
|
||||
process(city, datas)
|
||||
// updateOldRegion(city)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def process(city, results){
|
||||
// def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -70,7 +84,6 @@ class PoiWash {
|
||||
}
|
||||
|
||||
latch.await()
|
||||
|
||||
// return rets;
|
||||
}
|
||||
|
||||
@@ -85,10 +98,17 @@ class PoiWash {
|
||||
addr = addr.replace(data.city,'')
|
||||
}
|
||||
if(data.district){
|
||||
addr = addr.replace(data.district,'')
|
||||
if(!addr.contains(data.district+"路")){
|
||||
addr = addr.replace(data.district,'')
|
||||
}
|
||||
}
|
||||
if(data.township){
|
||||
addr = addr.replace(data.township,'')
|
||||
try{
|
||||
addr = addr.replace(data.township,'')
|
||||
}catch(Exception e){
|
||||
println data.id+" "+data.township
|
||||
}
|
||||
|
||||
}
|
||||
if(addr && addr.startsWith('-')){
|
||||
addr = addr.replace('---','').replace('--','').replace('-','')
|
||||
@@ -97,6 +117,12 @@ class PoiWash {
|
||||
return null;
|
||||
}
|
||||
|
||||
if(addr){
|
||||
for(def r : regions){
|
||||
addr = addr.replace(r, "")
|
||||
}
|
||||
}
|
||||
|
||||
if(addr){
|
||||
def towns = townsMap.get(data.fjw_city)
|
||||
for(def t : towns){
|
||||
@@ -115,9 +141,11 @@ class PoiWash {
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("name", '三明海大饲料有限公司')
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
@@ -125,7 +153,7 @@ class PoiWash {
|
||||
keys.put("address", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys)
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
@@ -133,13 +161,78 @@ class PoiWash {
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def updateOldRegion(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
keys.put("fjw_district", 1)
|
||||
keys.put("township", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
|
||||
def rets = []
|
||||
|
||||
def distMap = [:]
|
||||
for(def d : dbcursor){
|
||||
// def region = distMap.get(d.fjw_district)
|
||||
// if(!region){
|
||||
// distMap.put(d.fjw_district, d.district)
|
||||
// }
|
||||
//
|
||||
// if(d.district instanceof Double){
|
||||
// def newRegion = distMap.get(d.fjw_district)
|
||||
// if(newRegion){
|
||||
// println d.id +" "+d.fjw_district+" "+newRegion
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("district", newRegion)
|
||||
// MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
|
||||
if(d.township instanceof Double){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", "")
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
|
||||
println d.id
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllRegions(cityName){
|
||||
def q = new BasicDBObject()
|
||||
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
|
||||
q.put("province_name", cityName)
|
||||
}else{
|
||||
q.put("city_name", cityName)
|
||||
}
|
||||
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
|
||||
def set = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
set << d.area_name
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
|
||||
def findAllTownship(cityName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", cityName)
|
||||
def dbcursor = TOWNSHIP_CONN.find(q)
|
||||
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
|
||||
q.put("province_name", cityName)
|
||||
}else{
|
||||
q.put("city_name", cityName)
|
||||
}
|
||||
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = d.fangjia_city
|
||||
if(!city || city instanceof Double){
|
||||
city = RegionData.formatCity(d.city_name)
|
||||
}
|
||||
def towns = map.get(city)
|
||||
if(!towns){
|
||||
towns = []
|
||||
@@ -150,9 +243,64 @@ class PoiWash {
|
||||
if(alias){
|
||||
towns.addAll(alias)
|
||||
}
|
||||
towns.unique()
|
||||
// towns.unique()
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def check(){
|
||||
def citys = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市","三明市"]
|
||||
for(def city : citys){
|
||||
def datas = findPois(city)
|
||||
processUpdate(city, datas)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def processUpdate(city, results){
|
||||
// def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
if(map.address){
|
||||
def newAddr = DataUtil.toDBCcase(map.address)
|
||||
|
||||
if(newAddr.startsWith(";")){
|
||||
newAddr = newAddr.substring(1)
|
||||
}
|
||||
if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
|
||||
newAddr = ""
|
||||
}
|
||||
|
||||
if(map.address!=newAddr){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", newAddr)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
// return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user