Merge branch 'script_gj' into 'master'

Script gj

See merge request !862
This commit is contained in:
2024-06-18 19:22:45 +08:00
17 changed files with 1078 additions and 33 deletions
@@ -440,7 +440,7 @@ class CcbCommonUtils {
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","中爱花园")
// q.put("name","鸿运大院")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){
if(d['J']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
@@ -1301,17 +1301,17 @@ ExcelUtils.multiThreadingModify(fileName, {d->
// d['H'] = dist._id.toString()
// }
// }
if(!d['H'] && d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(!d['A'] && d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
d['H'] = dist.name
// d['H'] = dist.name
}
}
}
@@ -13,6 +13,7 @@ import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
@@ -276,7 +277,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
q.put("name","上唐山水")
// q.put("name","托斯卡纳庄园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -285,7 +286,7 @@ class CommonDatautils {
}
}
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -4573,6 +4574,28 @@ class CommonDatautils {
return townMap
}
static def findAllTownRanges(){
def q = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("town", 1)
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
def townMap = [:]
for(def d : dbcursor){
def lists = townMap.get(d.city)
if(!lists){
lists = []
}
def t = [:]
t.region = d.region
t.town = d.town
lists << t
townMap.put(d.city, lists)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
@@ -4634,6 +4657,7 @@ class CommonDatautils {
keys.put("district",1)
keys.put("street",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def regionMap = [:]
for(def d : dbcursor){
@@ -4655,6 +4679,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4662,6 +4687,7 @@ class CommonDatautils {
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
@@ -4678,6 +4704,34 @@ class CommonDatautils {
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city
def list = townMap.get(key)
if(!list){
list = []
}
def t = [:]
t.region = d.district
t.town = d.street
list << t
townMap.put(key, list)
}
return townMap
}
static def queryDistNameCity(){
def q = new BasicDBObject()
@@ -4748,6 +4802,54 @@ class CommonDatautils {
}
static def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
@@ -0,0 +1,312 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
static main(args) {
def task = new PoiLocation();
task.run();
}
def run(){
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
for(def city : citys){
findPois(city)
}
}
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("run_status", new BasicDBObject('$exists', false))
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
def index = 0
for(def d : dbcursor){
rets << d
if(rets.size()>=10000){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
updateToEs2(datas)
updateToMG(datas)
rets = []
print "."
}
}
return rets;
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
rets << map
if(map.update_status && map.update_status!=100){
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
}
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def running(d){
def regions = getFullRegion(d)
def town = null
def map =[:]
if(d.location){
//判断区域否在围栏里
def regionFlag = null
if(d.district){
for(def rs : regions){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
if(regionFlag){
//判断township是否在统计局中
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
if(town){
if(town.street == d.township){
//打标记
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = town.street
}
}else{
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
if (mapTown) {
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = mapTown.street
}else{
if(d.district!=gaodeMap.district){
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
}
}
}else{
//打标记“经纬度有误,不在区域围栏中”
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
return map;
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.district)){
names << r
}
}
}
return names;
}
def getSimilarTown(regions, city, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.update_status && map.update_status==2){
map.remove("run_status")
map.remove("update_status")
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def updateToMG(rows){
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def udata = new BasicDBObject()
m.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
def q = new BasicDBObject()
q.put("_id", m.id)
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
}
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
}
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
}
@@ -50,7 +50,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
synchTown()
// synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -61,6 +61,13 @@ class TownMiddleProcess {
// addDistrictDiffFlag()
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
}
@@ -800,4 +807,126 @@ class TownMiddleProcess {
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
});
}
def updateCommunityDiffStreet(){
def q = new BasicDBObject("type", "乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
def query = new BasicDBObject()
query.put("city_name", d.city)
query.put("area_name", d.district)
query.put("street_name", d.street)
def obj = COMMUNITY_CONN.findOne(query)
if(!obj){
query.put("street_name", d.district+d.street)
obj = COMMUNITY_CONN.findOne(query)
if(obj){
def newStreet = obj.street_name.replace(d.district, "")
def udata = new BasicDBObject()
udata.put("street_name", newStreet)
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
}else{
println d.city +" "+d.district+" "+d.street
}
}
}
}
def updateTownPoline(){
def city = "唐山市"
def name = "高新技术开发区街道"
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("city", city).append("street",name)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
def checkTown(){
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def ad : dbcursor){
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
def count = TOWNPOLYLINE_COLL.count(polyline_q)
if(count==1){
println ad.city+" "+ ad.street+" "+count
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("polyline_source", "区域+街道匹配")
def updateQuery = new BasicDBObject("_id", ad._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
def updateTownshipByRegion(){
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
def keys = new BasicDBObject()
keys.put("mzb_district_name",1)
keys.put("mzb_district_code",1)
keys.put("city",1)
keys.put("district",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
for(def d : dbcursor){
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
def udata = new BasicDBObject()
udata.put("mzb_district_name",d.mzb_district_name)
udata.put("mzb_district_code",d.mzb_district_code)
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
}
@@ -0,0 +1,324 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class TownshipStrandUpdate {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def fullCityMap = null
def regionMap = null
def commMap = null
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new TownshipStrandUpdate()
task.process()
}
void process() {
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
for(def city : citys){
if(["中山"].contains(city)){
continue
}
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
}
}
}
def checkData(def dists){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets << running(d)
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return dists
}
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def gaodeMap = null
def town = null
def mapTown = null
def oldSimlarTown = null
def newSimlarTown = null
if(d.lng){
d.lng = d.lng/10000000000.0
d.lat = d.lat/10000000000.0
if(d.township){
town = getSimilarTown(regions, fullCity, d.township)
}
if(town && town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
d.polygon_town = town.street
}
}
d.strand_street = ""
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
if(mapTown){
d.strand_street = mapTown.street
if(mapTown.shape && mapTown.shape.coordinates){
d.map_town_shape = "有"
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
if(flag){
d.map_polygon_town = mapTown.street
}
}
}else{
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
if(oldSimlarTown){
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
d.old_similar_town_score = oldSimlarTown.score
}
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
if(newSimlarTown){
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
d.new_similar_town_score = newSimlarTown.score
}
}
}
}
def mapCity = null
def mapRegion = null
def mapTownship = null
def mapAddr = null
if(gaodeMap){
if(gaodeMap.city && gaodeMap.city!="[]"){
mapCity = gaodeMap.city
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
}
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
mapAddr = gaodeMap.format_addr
}
}
d.mapCity = mapCity
d.mapRegion = mapRegion
d.mapTownship = mapTownship
d.mapAddr = mapAddr
if(mapCity && mapRegion){
if(mapCity == fullCity.city){
if(regions.contains(mapRegion)){
if(mapTownship){
if(town){
d.old_tjj_township = "是"
}else{
d.old_tjj_township = "否"
}
if(mapTown){
d.new_tjj_township = "是"
}else{
d.new_tjj_township = "否"
}
if(mapTownship == d.township){
if(town){
d.remark = "相同且为统计局乡镇"
}else{
d.remark = "非统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}else{
if(mapTown){
d.remark = "更正"
}else{
d.remark = "不同且不为统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}
}else{
d.remark = "地图无乡镇街道"
d.opt = "地图无街道信息"
}
}else{
d.remark = "区域不同"
d.opt = "经纬度可能跨区域"
}
}else{
d.remark = "城市不同"
d.opt = "经纬度可能跨城市"
}
}
if(d.map_town_shape && d.map_town_shape=="有"){
if(!d.map_polygon_town){
d.opt = "经纬度可疑"
}
}
return d;
}
def getSimilarTownByStr(fullCity, townstr, townList){
def maxScore = 0
def town = null
for(def townObj : townList){
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
if(score > maxScore){
maxScore = score
town = townObj
}
}
if(town){
town.score = maxScore
}
return town
}
def getSimilarTown(regions, fullCity, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(fullCity.city+"#"+r)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def getFullRegion(fullCity, d){
def regions = regionMap.get(fullCity.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region)){
names << r.district
}
}
}
return names;
}
def createDatas(dists){
def rets = []
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
for(def d : dists){
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
}
return rets;
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
}
@@ -1,13 +1,46 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class TradeDatasDistBatch {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
static main(args) {
// def
def task = new TradeDatasDistBatch()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("alias", d['D'])
alias_fang_COLL.save(udata)
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def flag = null
// if(!dist){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }else{
// flag = "名称匹配"
// }
// if(flag){
// d['F'] = dist.city
// d['G'] = dist.region
// d['H'] = dist.name
// }
})
}
}
@@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def stop = false
def month = -1
while(!stop){
if(sDate<202110){
if(sDate<202004){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
@@ -33,10 +33,10 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
// addGXDPrice(filePath)
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
addThirdPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
@@ -241,12 +241,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['L']) {
def dist = CommonDatautils.findDistrictById(d['E'])
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['L'] = gxdPrice
d['M'] = gxdPrice
}
}
})
@@ -270,16 +270,16 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['N']){
if(d['C'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['C'])
if(!d['L']){
if(d['B'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
// d['W'] = dbo.avg_price
// d['X'] = dbo.cal_type
d['N'] = dbo.avg_price_2b
d['L'] = dbo.avg_price_2b
}
}
}
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -80,13 +80,13 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
sysnchVillage()
// sysnchVillage()
// addTownship()
addTownship()
}
def addTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d->
if(d['E']){
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
// if(gaodeMap && gaodeMap.township){
@@ -114,7 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{
d['L'] = "是"
}
}
})
},0)
}
@@ -98,6 +98,23 @@ class InsidePolygon {
}
return list;
}
public static List<List<Point>> createPoint4(def pointList){
List<List<Point>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point> list = new ArrayList<>();
for(def arr : pointArr){
Point point = new Point();
double x = arr[0];
double y = arr[1];
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -76,7 +76,6 @@ class ElasticUtils {
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@@ -101,6 +100,30 @@ class ElasticUtils {
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 5
def nThread = Runtime.getRuntime().availableProcessors() * 4
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
/**
* 冪帠僅蛌華硊<br>