Merge branch 'script_gj' into 'master'

update

See merge request !1100
This commit is contained in:
2025-04-01 22:12:42 +08:00
12 changed files with 403 additions and 119 deletions
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\2025年02月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -5094,10 +5094,11 @@ class CommonDatautils {
def keys = new BasicDBObject()
keys.put("street",1)
keys.put("alias",1)
keys.put("shape",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
retMap.put(d.street, d.alias)
retMap.put(d.street, d)
}
return retMap
@@ -5,7 +5,11 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -17,6 +21,7 @@ class AutoAddOfficeBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
static main(args) {
runBatch(new AutoAddOfficeBatch(), args)
@@ -29,6 +34,16 @@ class AutoAddOfficeBatch extends BaseBatch{
// def newName = "开放广场".replaceAll(pattern, "")
// println newName
// findAddOfficeDatas() //查找需要新增的写字楼数据
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
addOffice()//新增写字楼
}
def findAddOfficeDatas(){
def citys = CommonDatautils.findAllCity()
def allAddOffices = []
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
@@ -46,6 +61,19 @@ class AutoAddOfficeBatch extends BaseBatch{
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
}
def checkInBaiduMapOffice(){
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx",{d->
if(!d['L']){
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
if(obj){
d['L'] = "百度存在"
d['M'] = obj.std_tag.toString()
}
}
},0)
}
def filterOffice(officeMaps){
def offices = []
def keys = []
@@ -171,4 +199,61 @@ class AutoAddOfficeBatch extends BaseBatch{
}
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def addOffice(){
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
if(d['L']!='人工删除'){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
if(city && region){
def locationstr = d['G'].toString()+","+d['H'].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
def udata = new BasicDBObject()
if(bdpoint){
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
}
udata.put("city", city)
udata.put("region", region)
udata.put("name", d['E'])
udata.put("address", d['I'])
udata.put("cat", "office")
udata.put("status", 0)
udata.put("c_date",new Date())
udata.put("m_date",new Date())
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
def estate_type = null
if(d['M'] && d['M'].contains("园区")){
estate_type = "创意园区"
}
if(!estate_type && d['E'].contains("创意")){
estate_type = "创意园区"
}
if(estate_type){
udata.put("estate_type2", estate_type)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map", markMap)
DbUtils.seaweed.save(udata)
println d['E'] +" "+udata._id.toString()
}
}
})
}
}
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* 名录集
*/
@@ -31,7 +29,7 @@ class MinglujiData {
}
def run(){
def prov = "河北省"
def prov = "省"
def city = "石家庄市"
// exportDatas(city)
// matchDev(prov, city)
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -32,11 +35,14 @@ class RegionStandardBatch extends BaseBatch{
@Override
void process(Object args) {
// seaweedRegionStandard()
poiRegionStandard()
// poiRegionStandard()
//通过乡镇街道修改区域 如:东莞市
// updateRegionByTown()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("district")
seaweedRegionStandardByCat("office")
}
def poiRegionStandard(){
@@ -93,7 +99,7 @@ class RegionStandardBatch extends BaseBatch{
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
def citys = ["伊春"]//CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
@@ -148,8 +154,12 @@ class RegionStandardBatch extends BaseBatch{
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
println city +" "+ districts.size()
def index = 0
for(def d : districts){
index ++
println index +" "+ d.region
if(set.contains(d.region)){
continue
}
@@ -157,8 +167,12 @@ class RegionStandardBatch extends BaseBatch{
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def obj = streetMap.get(key)
def alias = null
if(obj && obj.alias){
alias = obj.alias
}
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
@@ -166,13 +180,26 @@ class RegionStandardBatch extends BaseBatch{
flag = "pass"
break;
}
if(obj.shape){
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
if(inAreaFlag){
def newRegion = obj.street
newRegion = RegionData.formatRegion(d.city, newRegion)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
@@ -402,4 +429,27 @@ class RegionStandardBatch extends BaseBatch{
return rets;
}
def updateRegionByTown(){
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
def town = RegionData.formatRegion(d['B'], d['F'])
if(town){
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
if(!obj){
def udata = new BasicDBObject()
udata.put("region", town)
udata.put("m_date", new Date())
println d['F']+" "+" 区域修改为:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
println d['F']+" "+" 删除:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -219,34 +220,41 @@ class TempExportData {
}
def exportOfficeHouseCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "前瞻存在"
// }
// if(!d['T']){
// def obj = findYuanQu(d['B'], d['D'])
// def city = RegionData.formatCity(d['C'])
// def obj = findYuanQu(city, d['E'])
// if(obj){
// d['T'] = "前瞻存在"
// d['U'] = obj.name
// }
// }
// if(!d['T']) {
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
// def city = RegionData.formatCity(d['C'])
// def region = RegionData.formatRegion(city, d['D'])
//
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
// if (!count) {
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
// }
// d['S'] = count
// d['T'] = count
// }
if(!d['J']){
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
if(!d['T']){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
if(datas){
def sources = []
for(def o : datas){
sources << o.source
}
if(sources){
d['K'] = sources.join(",")
d['V'] = sources.join(",")
}
}
}
@@ -1,58 +1,57 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.math.RandomUtils
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
def city = "昌江"
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
def date = d['B']
def start = DateUtils.getFirstDayOfMonth(date)
def end = DateUtils.getLastDayOfMonth(date)
def weekends = DateUtils.getWeekendBetweenDates(start, end)
for(def weekend : weekends){
def avg_price = d['I']
def val = random(-200,200)
def new_price = (int)(avg_price*val/10000 + avg_price)
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
def o = new BasicDBObject()
o.put("avg_price", new_price)
o.put("c_date" , new Date())
o.put("city", city)
o.put("m_date" , new Date())
o.put("weekend", weekend)
o.put("cat","城市")
o.put("moving_average" , (int)avg_price)
if(!last_price){
o.put("weekend_markup" , 0)
}else{
def weekend_markup = (int)((new_price-last_price)*100/last_price)
o.put("weekend_markup" , weekend_markup)
}
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
},1)
public static final int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
import org.bson.types.ObjectId
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
Date creationDate = objectId.getDate();
System.out.println(creationDate);
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
//
//
//def city = "昌江"
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
// def date = d['B']
// def start = DateUtils.getFirstDayOfMonth(date)
// def end = DateUtils.getLastDayOfMonth(date)
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
// for(def weekend : weekends){
// def avg_price = d['I']
// def val = random(-200,200)
// def new_price = (int)(avg_price*val/10000 + avg_price)
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
//
// def o = new BasicDBObject()
// o.put("avg_price", new_price)
// o.put("c_date" , new Date())
// o.put("city", city)
// o.put("m_date" , new Date())
// o.put("weekend", weekend)
// o.put("cat","城市")
// o.put("moving_average" , (int)avg_price)
// if(!last_price){
// o.put("weekend_markup" , 0)
// }else{
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
// o.put("weekend_markup" , weekend_markup)
// }
//
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
// }
//},1)
//
//
//public static final int random(int i1, int i2) {
// int max = Math.max(i1, i2);
// int min = Math.min(i1, i2);
// return min + RandomUtils.nextInt(max - min + 1);
//}
//
//
//
//
//
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -45,6 +44,8 @@ class TownMiddleProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -95,7 +96,11 @@ class TownMiddleProcess {
// addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
addSpecialCityTownAlias()
// addSpecialCityTownAlias()
// updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline()
}
@@ -451,16 +456,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -681,7 +686,7 @@ class TownMiddleProcess {
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
@@ -1356,4 +1361,90 @@ class TownMiddleProcess {
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
def updateNewRegionPoyline(){
def regionPolylineMap = regionPolyline()
def q = new BasicDBObject()
q.put("type","区域")
// q.put("city", "苏州市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
for(def d : dbcursor){
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
println d.city+" "+d.district
if(shapeData){
println "新围栏数:"+shapeData.shape.coordinates.size()
}
if(d.shape){
println "旧围栏数:"+d.shape.coordinates.size()
}
if(d.shape && shapeData && shapeData.shape!=d.shape){
def udata = new BasicDBObject()
udata.shape = shapeData.shape
udata.location = shapeData.location
udata.m_date = new Date()
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
// if(Math.abs(area1-area)>=1){
// println "新:"+area
// println "旧:"+area1
// def udata = new BasicDBObject()
// udata.shape = shapeData.shape
// udata.location = shapeData.location
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// }
// }
}
}
}
def updateMappingPolyline(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("shape.type","polygon")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
updateAdministrativeMapping(d._id.toString(), d.shape)
}
}
def updateAdministrativeMapping(def id, def shape){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.shape = params.newshape",
"params":{
"newshape":{
"type":"${shape.type}",
"coordinates":${shape.coordinates}
}
}
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -3,7 +3,6 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -25,7 +24,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -52,9 +50,9 @@ class TownStandardBatch extends BaseBatch{
}
def TownStandard(){
// townStandardByCat("office")
townStandardByCat("office")
// townStandardByPoi()
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// updateTownData()
}
def townStandardByCat(def cat){
@@ -537,7 +535,7 @@ class TownStandardBatch extends BaseBatch{
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
@@ -1,16 +1,9 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -336,7 +329,7 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -5,6 +5,7 @@ import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.math.RandomUtils
@@ -17,8 +18,9 @@ import java.util.regex.Pattern
*/
class RegionPointCrawl {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new RegionPointCrawl();
@@ -26,16 +28,19 @@ class RegionPointCrawl {
//4个直辖市 北京市、上海市、天津市、重庆市
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
def provinces = ["甘肃省"]
def index = 0
for(def p : provinces){
index ++
println index +"/"+provinces.size() +" "+p
task.crawlCityAndDistrictCode(p);
println "城市 区域 code 完成"
task.run(p);
println "区域围栏 完成"
}
//已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省"
// def provinces = ["北京市","上海市","天津市","重庆市" ]
// def index = 0
// for(def p : provinces){
// index ++
// println index +"/"+provinces.size() +" "+p
// task.crawlCityAndDistrictCode(p);
// println "城市 区域 code 完成"
// task.run(p);
// println "区域围栏 完成"
// }
task.statSame()
}
@@ -47,7 +52,10 @@ class RegionPointCrawl {
def cityCodes = findCityCode(pro);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code);
def codeMap = findAllDistrictCode(cityMap.city_code, "district");
// if(!codeMap){
// codeMap = findAllDistrictCode(cityMap.city_code, "street");
// }
def index = 0
for(def m : codeMap){
index ++
@@ -93,7 +101,7 @@ class RegionPointCrawl {
def saveGaoDeRegionCode(province, cityName,districts){
if(districts){
for(def dist : districts){
if(dist.level=="district" || cityName=="东莞市"){
if(dist.level=="district" || dist.level=="street" ){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
@@ -125,10 +133,10 @@ class RegionPointCrawl {
return new BasicDBObject("$regex", pattern);
}
def findAllDistrictCode(def citycode){
def findAllDistrictCode(def citycode, def level){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("level", "district")
q.put("level", level)
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
@@ -149,6 +157,7 @@ class RegionPointCrawl {
def o = GAODECITYCODECOLL.findOne(q)
if(!o){
println name +" "+adcode+" 新增"
q.put("c_date", new Date())
q.put("center", center)
GAODECITYCODECOLL.save(q);
@@ -233,4 +242,62 @@ class RegionPointCrawl {
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
}
def statSame(){
def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def index = 0
for(def d : dbcursor){
index++
def q = new BasicDBObject()
q.put("city",d.city)
q.put("region", d.region)
def obj = OLD_POLYLINECOLL.findOne(q)
if(obj){
if(obj.polyline!=d.polyline){
println index +" "+ d.city +" "+d.region+" 2次区域不同"
}
}else{
println index +" "+ d.city +" "+d.region+" 没有此区域"
}
}
}
def regionPolyline(){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
}
@@ -3,15 +3,10 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
@@ -19,7 +14,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch extends BaseBatch{