update
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\2025年02月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+2
-1
@@ -5094,10 +5094,11 @@ class CommonDatautils {
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("street",1)
|
||||
keys.put("alias",1)
|
||||
keys.put("shape",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
retMap.put(d.street, d.alias)
|
||||
retMap.put(d.street, d)
|
||||
}
|
||||
|
||||
return retMap
|
||||
|
||||
+85
@@ -5,7 +5,11 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -17,6 +21,7 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AutoAddOfficeBatch(), args)
|
||||
@@ -29,6 +34,16 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
// def newName = "开放广场".replaceAll(pattern, "")
|
||||
// println newName
|
||||
|
||||
// findAddOfficeDatas() //查找需要新增的写字楼数据
|
||||
|
||||
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
|
||||
|
||||
addOffice()//新增写字楼
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findAddOfficeDatas(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allAddOffices = []
|
||||
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
|
||||
@@ -46,6 +61,19 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
|
||||
}
|
||||
|
||||
def checkInBaiduMapOffice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx",{d->
|
||||
if(!d['L']){
|
||||
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
|
||||
if(obj){
|
||||
d['L'] = "百度存在"
|
||||
d['M'] = obj.std_tag.toString()
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def filterOffice(officeMaps){
|
||||
def offices = []
|
||||
def keys = []
|
||||
@@ -171,4 +199,61 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
// q.put('std_tag', '房地产;住宅区')
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def addOffice(){
|
||||
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
|
||||
if(d['L']!='人工删除'){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(city && region){
|
||||
def locationstr = d['G'].toString()+","+d['H'].toString()
|
||||
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
|
||||
def udata = new BasicDBObject()
|
||||
if(bdpoint){
|
||||
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
|
||||
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
|
||||
}
|
||||
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
udata.put("name", d['E'])
|
||||
udata.put("address", d['I'])
|
||||
udata.put("cat", "office")
|
||||
udata.put("status", 0)
|
||||
udata.put("c_date",new Date())
|
||||
udata.put("m_date",new Date())
|
||||
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
|
||||
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
|
||||
|
||||
def estate_type = null
|
||||
if(d['M'] && d['M'].contains("园区")){
|
||||
estate_type = "创意园区"
|
||||
}
|
||||
|
||||
if(!estate_type && d['E'].contains("创意")){
|
||||
estate_type = "创意园区"
|
||||
}
|
||||
if(estate_type){
|
||||
udata.put("estate_type2", estate_type)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map", markMap)
|
||||
DbUtils.seaweed.save(udata)
|
||||
println d['E'] +" "+udata._id.toString()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1
-3
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 名录集
|
||||
*/
|
||||
@@ -31,7 +29,7 @@ class MinglujiData {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def prov = "河北省"
|
||||
def prov = "省"
|
||||
def city = "石家庄市"
|
||||
// exportDatas(city)
|
||||
// matchDev(prov, city)
|
||||
|
||||
+57
-7
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
@@ -32,11 +35,14 @@ class RegionStandardBatch extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// seaweedRegionStandard()
|
||||
poiRegionStandard()
|
||||
// poiRegionStandard()
|
||||
//通过乡镇街道修改区域 如:东莞市
|
||||
// updateRegionByTown()
|
||||
|
||||
}
|
||||
|
||||
def seaweedRegionStandard(){
|
||||
seaweedRegionStandardByCat("district")
|
||||
seaweedRegionStandardByCat("office")
|
||||
}
|
||||
def poiRegionStandard(){
|
||||
|
||||
@@ -93,7 +99,7 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
|
||||
def seaweedRegionStandardByCat(cat){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["伊春"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def rMap = regionMap.get(city)
|
||||
@@ -148,8 +154,12 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
|
||||
if(streetMap){
|
||||
set = streetMap.keySet()
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
|
||||
println city +" "+ districts.size()
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index ++
|
||||
println index +" "+ d.region
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
@@ -157,8 +167,12 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def updateRegion = null
|
||||
def flag = null
|
||||
for(def key : set){
|
||||
def alias = streetMap.get(key)
|
||||
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
|
||||
def obj = streetMap.get(key)
|
||||
def alias = null
|
||||
if(obj && obj.alias){
|
||||
alias = obj.alias
|
||||
}
|
||||
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
|
||||
def newRegion = RegionData.formatRegion(d.city, key)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
@@ -166,13 +180,26 @@ class RegionStandardBatch extends BaseBatch{
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
|
||||
if(obj.shape){
|
||||
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
|
||||
if(inAreaFlag){
|
||||
def newRegion = obj.street
|
||||
newRegion = RegionData.formatRegion(d.city, newRegion)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
}
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
if(flag!="pass"){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -402,4 +429,27 @@ class RegionStandardBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateRegionByTown(){
|
||||
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
|
||||
def town = RegionData.formatRegion(d['B'], d['F'])
|
||||
if(town){
|
||||
|
||||
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
|
||||
if(!obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", town)
|
||||
udata.put("m_date", new Date())
|
||||
println d['F']+" "+" 区域修改为:" + town
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("status", 1)
|
||||
println d['F']+" "+" 删除:" + town
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+16
-8
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -219,34 +220,41 @@ class TempExportData {
|
||||
}
|
||||
|
||||
def exportOfficeHouseCount(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
|
||||
// def obj = findYuanQu(d['B'], d['D'])
|
||||
// if(obj){
|
||||
// d['T'] = "前瞻存在"
|
||||
// }
|
||||
// if(!d['T']){
|
||||
// def obj = findYuanQu(d['B'], d['D'])
|
||||
// def city = RegionData.formatCity(d['C'])
|
||||
// def obj = findYuanQu(city, d['E'])
|
||||
// if(obj){
|
||||
// d['T'] = "前瞻存在"
|
||||
// d['U'] = obj.name
|
||||
// }
|
||||
// }
|
||||
// if(!d['T']) {
|
||||
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
|
||||
// def city = RegionData.formatCity(d['C'])
|
||||
// def region = RegionData.formatRegion(city, d['D'])
|
||||
//
|
||||
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
|
||||
// if (!count) {
|
||||
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
|
||||
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
|
||||
// }
|
||||
// d['S'] = count
|
||||
// d['T'] = count
|
||||
// }
|
||||
|
||||
if(!d['J']){
|
||||
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
|
||||
if(!d['T']){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
|
||||
if(datas){
|
||||
def sources = []
|
||||
for(def o : datas){
|
||||
sources << o.source
|
||||
}
|
||||
if(sources){
|
||||
d['K'] = sources.join(",")
|
||||
d['V'] = sources.join(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+52
-53
@@ -1,58 +1,57 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
|
||||
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
|
||||
|
||||
|
||||
def city = "昌江"
|
||||
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
|
||||
def date = d['B']
|
||||
def start = DateUtils.getFirstDayOfMonth(date)
|
||||
def end = DateUtils.getLastDayOfMonth(date)
|
||||
def weekends = DateUtils.getWeekendBetweenDates(start, end)
|
||||
for(def weekend : weekends){
|
||||
def avg_price = d['I']
|
||||
def val = random(-200,200)
|
||||
def new_price = (int)(avg_price*val/10000 + avg_price)
|
||||
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
|
||||
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
|
||||
|
||||
def o = new BasicDBObject()
|
||||
o.put("avg_price", new_price)
|
||||
o.put("c_date" , new Date())
|
||||
o.put("city", city)
|
||||
o.put("m_date" , new Date())
|
||||
o.put("weekend", weekend)
|
||||
o.put("cat","城市")
|
||||
o.put("moving_average" , (int)avg_price)
|
||||
if(!last_price){
|
||||
o.put("weekend_markup" , 0)
|
||||
}else{
|
||||
def weekend_markup = (int)((new_price-last_price)*100/last_price)
|
||||
o.put("weekend_markup" , weekend_markup)
|
||||
}
|
||||
|
||||
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
|
||||
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
|
||||
}
|
||||
},1)
|
||||
|
||||
|
||||
public static final int random(int i1, int i2) {
|
||||
int max = Math.max(i1, i2);
|
||||
int min = Math.min(i1, i2);
|
||||
return min + RandomUtils.nextInt(max - min + 1);
|
||||
}
|
||||
|
||||
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
|
||||
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
|
||||
Date creationDate = objectId.getDate();
|
||||
System.out.println(creationDate);
|
||||
|
||||
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
|
||||
//
|
||||
//
|
||||
//def city = "昌江"
|
||||
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
|
||||
// def date = d['B']
|
||||
// def start = DateUtils.getFirstDayOfMonth(date)
|
||||
// def end = DateUtils.getLastDayOfMonth(date)
|
||||
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
|
||||
// for(def weekend : weekends){
|
||||
// def avg_price = d['I']
|
||||
// def val = random(-200,200)
|
||||
// def new_price = (int)(avg_price*val/10000 + avg_price)
|
||||
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
|
||||
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
|
||||
//
|
||||
// def o = new BasicDBObject()
|
||||
// o.put("avg_price", new_price)
|
||||
// o.put("c_date" , new Date())
|
||||
// o.put("city", city)
|
||||
// o.put("m_date" , new Date())
|
||||
// o.put("weekend", weekend)
|
||||
// o.put("cat","城市")
|
||||
// o.put("moving_average" , (int)avg_price)
|
||||
// if(!last_price){
|
||||
// o.put("weekend_markup" , 0)
|
||||
// }else{
|
||||
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
|
||||
// o.put("weekend_markup" , weekend_markup)
|
||||
// }
|
||||
//
|
||||
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
|
||||
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
|
||||
// }
|
||||
//},1)
|
||||
//
|
||||
//
|
||||
//public static final int random(int i1, int i2) {
|
||||
// int max = Math.max(i1, i2);
|
||||
// int min = Math.min(i1, i2);
|
||||
// return min + RandomUtils.nextInt(max - min + 1);
|
||||
//}
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
+101
-10
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
@@ -45,6 +44,8 @@ class TownMiddleProcess {
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
@@ -95,7 +96,11 @@ class TownMiddleProcess {
|
||||
// addTownCenter()//百度接口添加中心点
|
||||
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||
|
||||
addSpecialCityTownAlias()
|
||||
// addSpecialCityTownAlias()
|
||||
|
||||
// updateNewRegionPoyline()//更新新的区域围栏
|
||||
|
||||
updateMappingPolyline()
|
||||
}
|
||||
|
||||
|
||||
@@ -451,16 +456,16 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
@@ -681,7 +686,7 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
@@ -1356,4 +1361,90 @@ class TownMiddleProcess {
|
||||
keys.put("street", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
|
||||
}
|
||||
|
||||
def updateNewRegionPoyline(){
|
||||
|
||||
def regionPolylineMap = regionPolyline()
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
// q.put("city", "苏州市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def d : dbcursor){
|
||||
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
|
||||
println d.city+" "+d.district
|
||||
if(shapeData){
|
||||
println "新围栏数:"+shapeData.shape.coordinates.size()
|
||||
}
|
||||
|
||||
if(d.shape){
|
||||
println "旧围栏数:"+d.shape.coordinates.size()
|
||||
}
|
||||
|
||||
if(d.shape && shapeData && shapeData.shape!=d.shape){
|
||||
def udata = new BasicDBObject()
|
||||
udata.shape = shapeData.shape
|
||||
udata.location = shapeData.location
|
||||
udata.m_date = new Date()
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
|
||||
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
|
||||
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
|
||||
// if(Math.abs(area1-area)>=1){
|
||||
// println "新:"+area
|
||||
// println "旧:"+area1
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.shape = shapeData.shape
|
||||
// udata.location = shapeData.location
|
||||
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateMappingPolyline(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("shape.type","polygon")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
updateAdministrativeMapping(d._id.toString(), d.shape)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateAdministrativeMapping(def id, def shape){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.shape = params.newshape",
|
||||
"params":{
|
||||
"newshape":{
|
||||
"type":"${shape.type}",
|
||||
"coordinates":${shape.coordinates}
|
||||
}
|
||||
}
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"_id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+3
-5
@@ -3,7 +3,6 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -25,7 +24,6 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
@@ -52,9 +50,9 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def TownStandard(){
|
||||
// townStandardByCat("office")
|
||||
townStandardByCat("office")
|
||||
// townStandardByPoi()
|
||||
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
// updateTownData()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
@@ -537,7 +535,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
// //通过调取地图api接口获取乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
// if(gaodeMap && gaodeMap.addressComponent) {
|
||||
// def addressComponent = gaodeMap.addressComponent
|
||||
// if(addressComponent){
|
||||
|
||||
+1
-8
@@ -1,16 +1,9 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.HttpRequestUtil
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
|
||||
@@ -336,7 +329,7 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+82
-15
@@ -5,6 +5,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
@@ -17,8 +18,9 @@ import java.util.regex.Pattern
|
||||
*/
|
||||
class RegionPointCrawl {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
@@ -26,16 +28,19 @@ class RegionPointCrawl {
|
||||
//4个直辖市 北京市、上海市、天津市、重庆市
|
||||
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
|
||||
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
|
||||
def provinces = ["甘肃省"]
|
||||
def index = 0
|
||||
for(def p : provinces){
|
||||
index ++
|
||||
println index +"/"+provinces.size() +" "+p
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
println "城市 区域 code 完成"
|
||||
task.run(p);
|
||||
println "区域围栏 完成"
|
||||
}
|
||||
//已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省"
|
||||
// def provinces = ["北京市","上海市","天津市","重庆市" ]
|
||||
// def index = 0
|
||||
// for(def p : provinces){
|
||||
// index ++
|
||||
// println index +"/"+provinces.size() +" "+p
|
||||
// task.crawlCityAndDistrictCode(p);
|
||||
// println "城市 区域 code 完成"
|
||||
// task.run(p);
|
||||
// println "区域围栏 完成"
|
||||
// }
|
||||
|
||||
task.statSame()
|
||||
|
||||
}
|
||||
|
||||
@@ -47,7 +52,10 @@ class RegionPointCrawl {
|
||||
def cityCodes = findCityCode(pro);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code, "district");
|
||||
// if(!codeMap){
|
||||
// codeMap = findAllDistrictCode(cityMap.city_code, "street");
|
||||
// }
|
||||
def index = 0
|
||||
for(def m : codeMap){
|
||||
index ++
|
||||
@@ -93,7 +101,7 @@ class RegionPointCrawl {
|
||||
def saveGaoDeRegionCode(province, cityName,districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district" || cityName=="东莞市"){
|
||||
if(dist.level=="district" || dist.level=="street" ){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
@@ -125,10 +133,10 @@ class RegionPointCrawl {
|
||||
return new BasicDBObject("$regex", pattern);
|
||||
}
|
||||
|
||||
def findAllDistrictCode(def citycode){
|
||||
def findAllDistrictCode(def citycode, def level){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("level", "district")
|
||||
q.put("level", level)
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
@@ -149,6 +157,7 @@ class RegionPointCrawl {
|
||||
|
||||
def o = GAODECITYCODECOLL.findOne(q)
|
||||
if(!o){
|
||||
println name +" "+adcode+" 新增"
|
||||
q.put("c_date", new Date())
|
||||
q.put("center", center)
|
||||
GAODECITYCODECOLL.save(q);
|
||||
@@ -233,4 +242,62 @@ class RegionPointCrawl {
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
}
|
||||
|
||||
def statSame(){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",d.city)
|
||||
q.put("region", d.region)
|
||||
def obj = OLD_POLYLINECOLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.polyline!=d.polyline){
|
||||
println index +" "+ d.city +" "+d.region+" 2次区域不同"
|
||||
}
|
||||
}else{
|
||||
println index +" "+ d.city +" "+d.region+" 没有此区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-8
@@ -3,15 +3,10 @@ package com.fangjia.script.gj.xiamen
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -19,7 +14,6 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
|
||||
Reference in New Issue
Block a user