Merge branch 'script_gj' into 'master'
Script gj See merge request !862
This commit is contained in:
+1
-1
@@ -440,7 +440,7 @@ class CcbCommonUtils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","中爱花园")
|
||||
// q.put("name","鸿运大院")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+8
-8
@@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['G']){
|
||||
if(d['J']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
@@ -1301,17 +1301,17 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['H'] && d['G'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(!d['A'] && d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
|
||||
d['H'] = dist.name
|
||||
// d['H'] = dist.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+104
-2
@@ -13,6 +13,7 @@ import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
@@ -276,7 +277,7 @@ class CommonDatautils {
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("name","上唐山水")
|
||||
// q.put("name","托斯卡纳庄园")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -285,7 +286,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -4573,6 +4574,28 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def findAllTownRanges(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("town", 1)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def lists = townMap.get(d.city)
|
||||
if(!lists){
|
||||
lists = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.region
|
||||
t.town = d.town
|
||||
lists << t
|
||||
townMap.put(d.city, lists)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
@@ -4634,6 +4657,7 @@ class CommonDatautils {
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
@@ -4655,6 +4679,7 @@ class CommonDatautils {
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4662,6 +4687,7 @@ class CommonDatautils {
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
@@ -4678,6 +4704,34 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.district
|
||||
t.town = d.street
|
||||
list << t
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -4748,6 +4802,54 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
|
||||
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
|
||||
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("run_status", new BasicDBObject('$exists', false))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>=10000){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
updateToEs2(datas)
|
||||
updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = running(d)
|
||||
rets << map
|
||||
if(map.update_status && map.update_status!=100){
|
||||
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def regions = getFullRegion(d)
|
||||
def town = null
|
||||
def map =[:]
|
||||
if(d.location){
|
||||
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if(d.district){
|
||||
for(def rs : regions){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
|
||||
if(regionFlag){
|
||||
//判断township是否在统计局中
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, d.city, d.township)
|
||||
}
|
||||
if(town){
|
||||
if(town.street == d.township){
|
||||
//打标记
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = town.street
|
||||
}
|
||||
}else{
|
||||
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
|
||||
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
|
||||
if (mapTown) {
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = mapTown.street
|
||||
}else{
|
||||
if(d.district!=gaodeMap.district){
|
||||
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
//打标记“经纬度有误,不在区域围栏中”
|
||||
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.district)){
|
||||
names << r
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def getSimilarTown(regions, city, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(city+"#"+r.district)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.township='${township}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}
|
||||
|
||||
|
||||
def updateToEs2(rows) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.update_status && map.update_status==2){
|
||||
map.remove("run_status")
|
||||
map.remove("update_status")
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def updateToMG(rows){
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def udata = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", m.id)
|
||||
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+130
-1
@@ -50,7 +50,7 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
synchTown()
|
||||
// synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
@@ -61,6 +61,13 @@ class TownMiddleProcess {
|
||||
|
||||
// addDistrictDiffFlag()
|
||||
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -800,4 +807,126 @@ class TownMiddleProcess {
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
});
|
||||
}
|
||||
|
||||
def updateCommunityDiffStreet(){
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city_name", d.city)
|
||||
query.put("area_name", d.district)
|
||||
query.put("street_name", d.street)
|
||||
def obj = COMMUNITY_CONN.findOne(query)
|
||||
if(!obj){
|
||||
query.put("street_name", d.district+d.street)
|
||||
obj = COMMUNITY_CONN.findOne(query)
|
||||
if(obj){
|
||||
def newStreet = obj.street_name.replace(d.district, "")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_name", newStreet)
|
||||
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
|
||||
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
|
||||
}else{
|
||||
println d.city +" "+d.district+" "+d.street
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updateTownPoline(){
|
||||
def city = "唐山市"
|
||||
def name = "高新技术开发区街道"
|
||||
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
|
||||
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("city", city).append("street",name)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def checkTown(){
|
||||
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def ad : dbcursor){
|
||||
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
|
||||
def count = TOWNPOLYLINE_COLL.count(polyline_q)
|
||||
if(count==1){
|
||||
println ad.city+" "+ ad.street+" "+count
|
||||
|
||||
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("polyline_source", "区域+街道匹配")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", ad._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateTownshipByRegion(){
|
||||
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("mzb_district_name",1)
|
||||
keys.put("mzb_district_code",1)
|
||||
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
|
||||
for(def d : dbcursor){
|
||||
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mzb_district_name",d.mzb_district_name)
|
||||
udata.put("mzb_district_code",d.mzb_district_code)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
def fullCityMap = null
|
||||
def regionMap = null
|
||||
def commMap = null
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
task.process()
|
||||
|
||||
}
|
||||
|
||||
|
||||
void process() {
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
if(["中山"].contains(city)){
|
||||
continue
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkData(def dists){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << running(d)
|
||||
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return dists
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
def oldSimlarTown = null
|
||||
def newSimlarTown = null
|
||||
if(d.lng){
|
||||
d.lng = d.lng/10000000000.0
|
||||
d.lat = d.lat/10000000000.0
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, fullCity, d.township)
|
||||
}
|
||||
|
||||
if(town && town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
d.polygon_town = town.street
|
||||
}
|
||||
}
|
||||
d.strand_street = ""
|
||||
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
|
||||
if(mapTown){
|
||||
d.strand_street = mapTown.street
|
||||
if(mapTown.shape && mapTown.shape.coordinates){
|
||||
d.map_town_shape = "有"
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
|
||||
if(flag){
|
||||
d.map_polygon_town = mapTown.street
|
||||
}
|
||||
}
|
||||
}else{
|
||||
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
|
||||
if(oldSimlarTown){
|
||||
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
|
||||
d.old_similar_town_score = oldSimlarTown.score
|
||||
}
|
||||
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
|
||||
if(newSimlarTown){
|
||||
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
|
||||
d.new_similar_town_score = newSimlarTown.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def mapCity = null
|
||||
def mapRegion = null
|
||||
def mapTownship = null
|
||||
def mapAddr = null
|
||||
if(gaodeMap){
|
||||
if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
mapCity = gaodeMap.city
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
}
|
||||
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
mapAddr = gaodeMap.format_addr
|
||||
}
|
||||
}
|
||||
d.mapCity = mapCity
|
||||
d.mapRegion = mapRegion
|
||||
d.mapTownship = mapTownship
|
||||
d.mapAddr = mapAddr
|
||||
if(mapCity && mapRegion){
|
||||
if(mapCity == fullCity.city){
|
||||
if(regions.contains(mapRegion)){
|
||||
if(mapTownship){
|
||||
if(town){
|
||||
d.old_tjj_township = "是"
|
||||
}else{
|
||||
d.old_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTown){
|
||||
d.new_tjj_township = "是"
|
||||
}else{
|
||||
d.new_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTownship == d.township){
|
||||
if(town){
|
||||
d.remark = "相同且为统计局乡镇"
|
||||
}else{
|
||||
d.remark = "非统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}else{
|
||||
if(mapTown){
|
||||
d.remark = "更正"
|
||||
}else{
|
||||
d.remark = "不同且不为统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d.remark = "地图无乡镇街道"
|
||||
d.opt = "地图无街道信息"
|
||||
}
|
||||
|
||||
}else{
|
||||
d.remark = "区域不同"
|
||||
d.opt = "经纬度可能跨区域"
|
||||
}
|
||||
}else{
|
||||
d.remark = "城市不同"
|
||||
d.opt = "经纬度可能跨城市"
|
||||
}
|
||||
}
|
||||
|
||||
if(d.map_town_shape && d.map_town_shape=="有"){
|
||||
if(!d.map_polygon_town){
|
||||
d.opt = "经纬度可疑"
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def getSimilarTownByStr(fullCity, townstr, townList){
|
||||
|
||||
def maxScore = 0
|
||||
def town = null
|
||||
for(def townObj : townList){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
|
||||
if(score > maxScore){
|
||||
maxScore = score
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
town.score = maxScore
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getSimilarTown(regions, fullCity, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(fullCity.city+"#"+r)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getFullRegion(fullCity, d){
|
||||
def regions = regionMap.get(fullCity.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
|
||||
def createDatas(dists){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
|
||||
for(def d : dists){
|
||||
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -1,13 +1,46 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class TradeDatasDistBatch {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
// def
|
||||
def task = new TradeDatasDistBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
udata.put("name", d['C'])
|
||||
udata.put("alias", d['D'])
|
||||
alias_fang_COLL.save(udata)
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def flag = null
|
||||
// if(!dist){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }else{
|
||||
// flag = "名称匹配"
|
||||
// }
|
||||
// if(flag){
|
||||
// d['F'] = dist.city
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
if(sDate<202110){
|
||||
if(sDate<202004){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
|
||||
+10
-10
@@ -33,10 +33,10 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
@@ -241,12 +241,12 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['L']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['L'] = gxdPrice
|
||||
d['M'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -270,16 +270,16 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['N']){
|
||||
if(d['C'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(!d['L']){
|
||||
if(d['B'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
// d['W'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
d['N'] = dbo.avg_price_2b
|
||||
d['L'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
|
||||
'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
|
||||
+4
-4
@@ -80,13 +80,13 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
sysnchVillage()
|
||||
// sysnchVillage()
|
||||
|
||||
// addTownship()
|
||||
addTownship()
|
||||
}
|
||||
|
||||
def addTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d->
|
||||
if(d['E']){
|
||||
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
@@ -114,7 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
d['L'] = "是"
|
||||
}
|
||||
}
|
||||
})
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
|
||||
+17
@@ -98,6 +98,23 @@ class InsidePolygon {
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<List<Point>> createPoint4(def pointList){
|
||||
List<List<Point>> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
List<Point> list = new ArrayList<>();
|
||||
for(def arr : pointArr){
|
||||
Point point = new Point();
|
||||
double x = arr[0];
|
||||
double y = arr[1];
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个经纬度点与区域的关系
|
||||
|
||||
+24
-1
@@ -76,7 +76,6 @@ class ElasticUtils {
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@@ -101,6 +100,30 @@ class ElasticUtils {
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
Reference in New Issue
Block a user