diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index dfd784e71..de2216df9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -8,7 +8,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\2025年02月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e6e2d944b..33b711d5d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -5094,10 +5094,11 @@ class CommonDatautils { def keys = new BasicDBObject() keys.put("street",1) keys.put("alias",1) + keys.put("shape",1) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] for(def d : dbcursor){ - retMap.put(d.street, d.alias) + retMap.put(d.street, d) } return retMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy index cfa15156b..727e3cafc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy @@ -5,7 +5,11 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -17,6 +21,7 @@ class AutoAddOfficeBatch extends BaseBatch{ def fullCityMap = CommonDatautils.queryFullNameByCity() public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) static main(args) { runBatch(new AutoAddOfficeBatch(), args) @@ -29,6 +34,16 @@ class AutoAddOfficeBatch extends BaseBatch{ // def newName = "开放广场".replaceAll(pattern, "") // println newName +// findAddOfficeDatas() //查找需要新增的写字楼数据 + +// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据 + + addOffice()//新增写字楼 + + + } + + def findAddOfficeDatas(){ def citys = CommonDatautils.findAllCity() def allAddOffices = [] allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"] @@ -46,6 +61,19 @@ class AutoAddOfficeBatch extends BaseBatch{ ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据") } + def checkInBaiduMapOffice(){ + ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx",{d-> + if(!d['L']){ + def obj = findBaiduDatas(d['C'], d['D'], d['E']) + if(obj){ + d['L'] = "百度存在" + d['M'] = obj.std_tag.toString() + } + } + + },0) + } + def filterOffice(officeMaps){ def offices = [] def keys = [] @@ -171,4 +199,61 @@ class AutoAddOfficeBatch extends BaseBatch{ } return null } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) +// q.put('std_tag', '房地产;住宅区') + def obj = BAIDU_POI_COLL.findOne(q) + return obj; + } + + def addOffice(){ + ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d-> + if(d['L']!='人工删除'){ + def city = RegionData.formatCity(d['C']) + def region = RegionData.formatRegion(city, d['D']) + if(city && region){ + def locationstr = d['G'].toString()+","+d['H'].toString() + def bdpoint = FangjiaClientUtils.convert(locationstr, 1) + def udata = new BasicDBObject() + if(bdpoint){ + udata.put("lng2", (bdpoint[0]*10000000000).longValue()) + udata.put("lat2", (bdpoint[1]*10000000000).longValue()) + } + + udata.put("city", city) + udata.put("region", region) + udata.put("name", d['E']) + udata.put("address", d['I']) + udata.put("cat", "office") + udata.put("status", 0) + udata.put("c_date",new Date()) + udata.put("m_date",new Date()) + udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue()) + udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue()) + + def estate_type = null + if(d['M'] && d['M'].contains("园区")){ + estate_type = "创意园区" + } + + if(!estate_type && d['E'].contains("创意")){ + estate_type = "创意园区" + } + if(estate_type){ + udata.put("estate_type2", estate_type) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + udata.put("mark_map", markMap) + DbUtils.seaweed.save(udata) + println d['E'] +" "+udata._id.toString() + } + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 1c94475bc..2cc1b9057 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.impl.AMQImpl import java.util.regex.Pattern - /** * 名录集 */ @@ -31,7 +29,7 @@ class MinglujiData { } def run(){ - def prov = "河北省" + def prov = "省" def city = "石家庄市" // exportDatas(city) // matchDev(prov, city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index abce57fe0..34f1b2c1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -4,9 +4,12 @@ import com.fangjia.core.RegionData import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import org.apache.commons.io.FileUtils +import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions @@ -32,11 +35,14 @@ class RegionStandardBatch extends BaseBatch{ @Override void process(Object args) { // seaweedRegionStandard() - poiRegionStandard() +// poiRegionStandard() + //通过乡镇街道修改区域 如:东莞市 +// updateRegionByTown() + } def seaweedRegionStandard(){ - seaweedRegionStandardByCat("district") + seaweedRegionStandardByCat("office") } def poiRegionStandard(){ @@ -93,7 +99,7 @@ class RegionStandardBatch extends BaseBatch{ def seaweedRegionStandardByCat(cat){ - def citys = CommonDatautils.findAllCity() + def citys = ["伊春"]//CommonDatautils.findAllCity() for(def city : citys){ def rMap = regionMap.get(city) @@ -148,8 +154,12 @@ class RegionStandardBatch extends BaseBatch{ def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName) if(streetMap){ set = streetMap.keySet() - def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"]) + println city +" "+ districts.size() + def index = 0 for(def d : districts){ + index ++ + println index +" "+ d.region if(set.contains(d.region)){ continue } @@ -157,8 +167,12 @@ class RegionStandardBatch extends BaseBatch{ def updateRegion = null def flag = null for(def key : set){ - def alias = streetMap.get(key) - if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){ + def obj = streetMap.get(key) + def alias = null + if(obj && obj.alias){ + alias = obj.alias + } + if((alias && alias.contains(d.region)) || (!alias && key==d.region)){ def newRegion = RegionData.formatRegion(d.city, key) if(newRegion!=d.region){ updateRegion = newRegion @@ -166,13 +180,26 @@ class RegionStandardBatch extends BaseBatch{ flag = "pass" break; } + + if(obj.shape){ + def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates)) + if(inAreaFlag){ + def newRegion = obj.street + newRegion = RegionData.formatRegion(d.city, newRegion) + if(newRegion!=d.region){ + updateRegion = newRegion + } + flag = "pass" + break; + } + } } if(updateRegion){ def udata = new BasicDBObject() udata.put("region", updateRegion) udata.put("m_date", new Date()) println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion -// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) }else{ if(flag!="pass"){ def udata = new BasicDBObject() @@ -402,4 +429,27 @@ class RegionStandardBatch extends BaseBatch{ return rets; } + def updateRegionByTown(){ + ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d-> + def town = RegionData.formatRegion(d['B'], d['F']) + if(town){ + + def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office") + if(!obj){ + def udata = new BasicDBObject() + udata.put("region", town) + udata.put("m_date", new Date()) + println d['F']+" "+" 区域修改为:" + town + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("status", 1) + println d['F']+" "+" 删除:" + town + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + } + + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy index b5f4ceb7f..c3d0bf761 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -219,34 +220,41 @@ class TempExportData { } def exportOfficeHouseCount(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d-> // def obj = findYuanQu(d['B'], d['D']) // if(obj){ // d['T'] = "前瞻存在" // } // if(!d['T']){ -// def obj = findYuanQu(d['B'], d['D']) +// def city = RegionData.formatCity(d['C']) +// def obj = findYuanQu(city, d['E']) // if(obj){ +// d['T'] = "前瞻存在" // d['U'] = obj.name // } // } // if(!d['T']) { -// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell") +// def city = RegionData.formatCity(d['C']) +// def region = RegionData.formatRegion(city, d['D']) +// +// def count = findHouseCountByCat(city, region, d['E'], "office_sell") // if (!count) { -// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent") +// count = findHouseCountByCat(city, region, d['E'], "office_rent") // } -// d['S'] = count +// d['T'] = count // } - if(!d['J']){ - def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D']) + if(!d['T']){ + def city = RegionData.formatCity(d['C']) + def region = RegionData.formatRegion(city, d['D']) + def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E']) if(datas){ def sources = [] for(def o : datas){ sources << o.source } if(sources){ - d['K'] = sources.join(",") + d['V'] = sources.join(",") } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy index 3f08229ce..1d22f4cc3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy @@ -1,58 +1,57 @@ package com.fangjia.script.gj.data_handle -import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection -import org.apache.commons.lang.math.RandomUtils - -def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; -DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl) - - -def city = "昌江" -ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d-> - def date = d['B'] - def start = DateUtils.getFirstDayOfMonth(date) - def end = DateUtils.getLastDayOfMonth(date) - def weekends = DateUtils.getWeekendBetweenDates(start, end) - for(def weekend : weekends){ - def avg_price = d['I'] - def val = random(-200,200) - def new_price = (int)(avg_price*val/10000 + avg_price) - def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7) - def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend) - - def o = new BasicDBObject() - o.put("avg_price", new_price) - o.put("c_date" , new Date()) - o.put("city", city) - o.put("m_date" , new Date()) - o.put("weekend", weekend) - o.put("cat","城市") - o.put("moving_average" , (int)avg_price) - if(!last_price){ - o.put("weekend_markup" , 0) - }else{ - def weekend_markup = (int)((new_price-last_price)*100/last_price) - o.put("weekend_markup" , weekend_markup) - } - - def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend) - REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false) - } -},1) - - -public static final int random(int i1, int i2) { - int max = Math.max(i1, i2); - int min = Math.min(i1, i2); - return min + RandomUtils.nextInt(max - min + 1); -} - - +import org.bson.types.ObjectId +ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40"); +Date creationDate = objectId.getDate(); +System.out.println(creationDate); +//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; +//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl) +// +// +//def city = "昌江" +//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d-> +// def date = d['B'] +// def start = DateUtils.getFirstDayOfMonth(date) +// def end = DateUtils.getLastDayOfMonth(date) +// def weekends = DateUtils.getWeekendBetweenDates(start, end) +// for(def weekend : weekends){ +// def avg_price = d['I'] +// def val = random(-200,200) +// def new_price = (int)(avg_price*val/10000 + avg_price) +// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7) +// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend) +// +// def o = new BasicDBObject() +// o.put("avg_price", new_price) +// o.put("c_date" , new Date()) +// o.put("city", city) +// o.put("m_date" , new Date()) +// o.put("weekend", weekend) +// o.put("cat","城市") +// o.put("moving_average" , (int)avg_price) +// if(!last_price){ +// o.put("weekend_markup" , 0) +// }else{ +// def weekend_markup = (int)((new_price-last_price)*100/last_price) +// o.put("weekend_markup" , weekend_markup) +// } +// +// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend) +// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false) +// } +//},1) +// +// +//public static final int random(int i1, int i2) { +// int max = Math.max(i1, i2); +// int min = Math.min(i1, i2); +// return min + RandomUtils.nextInt(max - min + 1); +//} +// +// +// +// +// diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 4ecd49ed0..68ca7a765 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.BaiduMapUtils -import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ProvinceData import com.mongodb.AggregationOutput import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject -import com.mongodb.client.AggregateIterable -import org.bson.Document +import net.sf.json.JSONObject import org.bson.types.ObjectId import java.util.regex.Pattern - /** * 乡镇街道中间表 * 包含市,区,乡镇街道别名,来源 统计局community表 @@ -45,6 +44,8 @@ class TownMiddleProcess { DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025") + def fullCityMap = CommonDatautils.queryFullNameCity() static main(args) { @@ -95,7 +96,11 @@ class TownMiddleProcess { // addTownCenter()//百度接口添加中心点 // checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中 - addSpecialCityTownAlias() +// addSpecialCityTownAlias() + +// updateNewRegionPoyline()//更新新的区域围栏 + + updateMappingPolyline() } @@ -451,16 +456,16 @@ class TownMiddleProcess { for(def d : dbcursor){ datas << d if (datas.size() >= 2000) { -// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); - ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); +// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) println datas.size() datas.clear(); } } if (datas.size() >= 0) { -// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); - ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); +// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) datas.clear(); } } @@ -681,7 +686,7 @@ class TownMiddleProcess { } def regionPolyline(){ - def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district")) + def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district")) def map = [:] for(def d : dbcursor){ def resultMap = [:] @@ -1356,4 +1361,90 @@ class TownMiddleProcess { keys.put("street", 1) return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList() } + + def updateNewRegionPoyline(){ + + def regionPolylineMap = regionPolyline() + def q = new BasicDBObject() + q.put("type","区域") +// q.put("city", "苏州市") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + for(def d : dbcursor){ + def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district) + println d.city+" "+d.district + if(shapeData){ + println "新围栏数:"+shapeData.shape.coordinates.size() + } + + if(d.shape){ + println "旧围栏数:"+d.shape.coordinates.size() + } + + if(d.shape && shapeData && shapeData.shape!=d.shape){ + def udata = new BasicDBObject() + udata.shape = shapeData.shape + udata.location = shapeData.location + udata.m_date = new Date() + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) +// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){ +// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0]) +// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0]) +// if(Math.abs(area1-area)>=1){ +// println "新:"+area +// println "旧:"+area1 +// def udata = new BasicDBObject() +// udata.shape = shapeData.shape +// udata.location = shapeData.location +// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) +// } +// } + } + } + } + + def updateMappingPolyline(){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("shape.type","polygon") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + for(def d : dbcursor){ + updateAdministrativeMapping(d._id.toString(), d.shape) + } + } + + + def updateAdministrativeMapping(def id, def shape){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.shape = params.newshape", + "params":{ + "newshape":{ + "type":"${shape.type}", + "coordinates":${shape.coordinates} + } + } + }, + "query": { + "term": { + "_id": "${id}" + } + } + } + """ + HttpRequest request = new HttpRequest() + try{ + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = JSONObject.fromObject(res) + println(response) + }catch(Exception e){ + println "error : "+e.getMessage() + } + + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 69c368290..92c0cda41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -3,7 +3,6 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 -import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.BaiduMapUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -25,7 +24,6 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - /** * 区域标准化 * 1.小区库乡镇街道标准化,包含小区,商铺,写字楼 @@ -52,9 +50,9 @@ class TownStandardBatch extends BaseBatch{ } def TownStandard(){ -// townStandardByCat("office") + townStandardByCat("office") // townStandardByPoi() - addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 +// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 // updateTownData() } def townStandardByCat(def cat){ @@ -537,7 +535,7 @@ class TownStandardBatch extends BaseBatch{ // //通过调取地图api接口获取乡镇街道 // if(!saveObj.township){ -// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon) + def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon) // if(gaodeMap && gaodeMap.addressComponent) { // def addressComponent = gaodeMap.addressComponent // if(addressComponent){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 35179b3d9..102e4666e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -1,16 +1,9 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest -import com.fangjia.script.addrepo.InvokeHttpRequest -import com.fangjia.script.statistics.poi.AmapPoi -import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils -import com.fangjia.script.util.HttpRequestUtil -import com.mongodb.DBObject import net.sf.json.JSONArray import net.sf.json.JSONObject -import org.apache.commons.collections.CollectionUtils -import org.apache.commons.lang.StringUtils class FangjiaClientUtils { @@ -336,7 +329,7 @@ class FangjiaClientUtils { try { address = address.replace("{","") // def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") - def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy index e80dd9c35..030658e63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy @@ -5,6 +5,7 @@ import com.fangjia.core.RegionData import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.math.RandomUtils @@ -17,8 +18,9 @@ import java.util.regex.Pattern */ class RegionPointCrawl { - static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025") static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") + static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static main(args) { def task = new RegionPointCrawl(); @@ -26,16 +28,19 @@ class RegionPointCrawl { //4个直辖市 北京市、上海市、天津市、重庆市 //5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区" // def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"] - def provinces = ["甘肃省"] - def index = 0 - for(def p : provinces){ - index ++ - println index +"/"+provinces.size() +" "+p - task.crawlCityAndDistrictCode(p); - println "城市 区域 code 完成" - task.run(p); - println "区域围栏 完成" - } + //已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省" +// def provinces = ["北京市","上海市","天津市","重庆市" ] +// def index = 0 +// for(def p : provinces){ +// index ++ +// println index +"/"+provinces.size() +" "+p +// task.crawlCityAndDistrictCode(p); +// println "城市 区域 code 完成" +// task.run(p); +// println "区域围栏 完成" +// } + + task.statSame() } @@ -47,7 +52,10 @@ class RegionPointCrawl { def cityCodes = findCityCode(pro); if(cityCodes){ for(def cityMap : cityCodes){ - def codeMap = findAllDistrictCode(cityMap.city_code); + def codeMap = findAllDistrictCode(cityMap.city_code, "district"); +// if(!codeMap){ +// codeMap = findAllDistrictCode(cityMap.city_code, "street"); +// } def index = 0 for(def m : codeMap){ index ++ @@ -93,7 +101,7 @@ class RegionPointCrawl { def saveGaoDeRegionCode(province, cityName,districts){ if(districts){ for(def dist : districts){ - if(dist.level=="district" || cityName=="东莞市"){ + if(dist.level=="district" || dist.level=="street" ){ saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); } } @@ -125,10 +133,10 @@ class RegionPointCrawl { return new BasicDBObject("$regex", pattern); } - def findAllDistrictCode(def citycode){ + def findAllDistrictCode(def citycode, def level){ def q = new BasicDBObject() q.put("city_code", citycode) - q.put("level", "district") + q.put("level", level) def dbcursor = GAODECITYCODECOLL.find(q) def rets = [] for(def d : dbcursor){ @@ -149,6 +157,7 @@ class RegionPointCrawl { def o = GAODECITYCODECOLL.findOne(q) if(!o){ + println name +" "+adcode+" 新增" q.put("c_date", new Date()) q.put("center", center) GAODECITYCODECOLL.save(q); @@ -233,4 +242,62 @@ class RegionPointCrawl { def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); } + + def statSame(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + + def index = 0 + for(def d : dbcursor){ + index++ + def q = new BasicDBObject() + q.put("city",d.city) + q.put("region", d.region) + def obj = OLD_POLYLINECOLL.findOne(q) + if(obj){ + if(obj.polyline!=d.polyline){ + println index +" "+ d.city +" "+d.region+" 2次区域不同" + } + }else{ + println index +" "+ d.city +" "+d.region+" 没有此区域" + } + } + } + + + def regionPolyline(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district")) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + + def fullCity = fullCityMap.get(d.city) + map.put(fullCity.city+"|@|"+d.region, udata) + } + return map; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 41ccc9e38..d92ceef54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -3,15 +3,10 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.util.GeoUtils import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.DateUtil -import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.ProvinceData +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.* import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.AMQP import org.bson.types.ObjectId import java.util.concurrent.CopyOnWriteArrayList @@ -19,7 +14,6 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong -import com.fangjia.script.common.DataHandleUtils class SeaweedDataSynch extends BaseBatch{