diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index aa9eca228..7a4062270 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -440,7 +440,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","中爱花园") +// q.put("name","鸿运大院") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 14c61c7e6..c43a791b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 019c22494..d527b08dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['G']){ + if(d['J']){ // def region = RegionData.formatRegion(d['B'], d['C']) // if(region){ // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") @@ -1301,17 +1301,17 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['H'] && d['G'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['G']) + if(!d['A'] && d['J'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ -// d['A'] = dist.city -// d['B'] = dist.region -// d['C'] = dist.name + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat - d['H'] = dist.name +// d['H'] = dist.name } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 634bf7b80..e9fa92613 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -13,6 +13,7 @@ import com.mongodb.AggregationOutput import com.mongodb.client.MongoCursor import com.mongodb.client.model.Accumulators import com.mongodb.client.model.Aggregates +import com.rabbitmq.client.impl.AMQImpl import groovy.json.JsonSlurper import net.sf.json.JSONArray @@ -276,7 +277,7 @@ class CommonDatautils { q.put("status", 0); q.put("cat", cat); q.put("city", city); - q.put("name","上唐山水") +// q.put("name","托斯卡纳庄园") DBObject keys = new BasicDBObject(); if(fields){ @@ -285,7 +286,7 @@ class CommonDatautils { } } - def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) + def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000) List rets = [] for(def obj : cursor){ rets << obj @@ -4573,6 +4574,28 @@ class CommonDatautils { return townMap } + static def findAllTownRanges(){ + def q = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("town", 1) + def dbcursor = TOWN_POLYLINECOLL.find(q, keys) + def townMap = [:] + for(def d : dbcursor){ + def lists = townMap.get(d.city) + if(!lists){ + lists = [] + } + def t = [:] + t.region = d.region + t.town = d.town + lists << t + townMap.put(d.city, lists) + } + return townMap + } + static def queryFullNameCity(){ def q = new BasicDBObject() def dbcursor = administrative_division_100_CONN.find(q) @@ -4634,6 +4657,7 @@ class CommonDatautils { keys.put("district",1) keys.put("street",1) keys.put("mzb_district_name", 1) + keys.put("shape", 1) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000) def regionMap = [:] for(def d : dbcursor){ @@ -4655,6 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") + q.put("province",new BasicDBObject('$in', ['福建省','重庆市'])) def keys = new BasicDBObject() keys.put("province",1) @@ -4662,6 +4687,7 @@ class CommonDatautils { keys.put("district",1) keys.put("street",1) keys.put("shape",1) + keys.put("alias",1) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) def townMap = [:] for(def d : dbcursor){ @@ -4678,6 +4704,34 @@ class CommonDatautils { } + static def queryNewTownByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city + def list = townMap.get(key) + if(!list){ + list = [] + } + def t = [:] + t.region = d.district + t.town = d.street + list << t + townMap.put(key, list) + } + return townMap + + } + + static def queryDistNameCity(){ def q = new BasicDBObject() @@ -4748,6 +4802,54 @@ class CommonDatautils { } + static def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy new file mode 100644 index 000000000..e46528404 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -0,0 +1,312 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PoiLocation { + + def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity() + def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity() + def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges() + def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity() + + static main(args) { + def task = new PoiLocation(); + task.run(); + } + def run(){ + def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"] + for(def city : citys){ + findPois(city) + } + } + + + def findPois(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("run_status", new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("_id", 1) + keys.put("province", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("township", 1) + keys.put("location", 1) + + def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def rets = [] + def index = 0 + for(def d : dbcursor){ + rets << d + if(rets.size()>=10000){ + index++ + def datas = checkData(index, rets) +// updateToEs(datas) + updateToEs2(datas) + updateToMG(datas) + rets = [] + print "." + } + } + return rets; + } + + def checkData(def index, def datas){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + + def updateToMG(rows){ + def rets = [] + for(def i=0;i + if(k!="id"){ + udata.put(k,v) + } + } + def q = new BasicDBObject() + q.put("_id", m.id) + MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy new file mode 100644 index 000000000..4d77b4cdf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels + +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class Test{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) +// 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + + static main(args) { + def task = new Test() + task.run() + } + + def run(){ + def i=1 + while(i<2){ + def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); + def rets = process(districts) + println i + " "+districts.size() +" "+rets.size() + i++ + } + + } + + def process(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+d.town+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + + + def updateQuery = new BasicDBObject("city", city).append("street",name) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + + def checkTown(){ + def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + for(def ad : dbcursor){ + def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street) + def count = TOWNPOLYLINE_COLL.count(polyline_q) + if(count==1){ + println ad.city+" "+ ad.street+" "+count + + def d = TOWNPOLYLINE_COLL.findOne(polyline_q) + if(d){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("polyline_source", "区域+街道匹配") + + def updateQuery = new BasicDBObject("_id", ad._id) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + } + } + + + def updateTownshipByRegion(){ + def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域") + def keys = new BasicDBObject() + keys.put("mzb_district_name",1) + keys.put("mzb_district_code",1) + + keys.put("city",1) + keys.put("district",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys) + for(def d : dbcursor){ + def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city) + def udata = new BasicDBObject() + udata.put("mzb_district_name",d.mzb_district_name) + udata.put("mzb_district_code",d.mzb_district_code) + ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true) + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy new file mode 100644 index 000000000..f6e830017 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -0,0 +1,324 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TownshipStrandUpdate { + + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + def fullCityMap = null + def regionMap = null + def commMap = null + def polylineMap = null + def newAdvMappingMap = null + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new TownshipStrandUpdate() + task.process() + + } + + + void process() { + fullCityMap = CommonDatautils.queryFullNameByCity() + regionMap = CommonDatautils.queryDistNameByCity() + commMap = CommonDatautils.queryCommunityByCity() + polylineMap = CommonDatautils.findAllTownRanges() + newAdvMappingMap = CommonDatautils.queryNewTownByCity() +// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + for(def city : citys){ + if(["中山"].contains(city)){ + continue + } + def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + if(!file.exists()){ + def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) + dists = checkData(dists) + def datas = createDatas(dists) + ExcelUtils.write(fileName, datas, "${city}") + } + } + } + + def checkData(def dists){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i maxScore){ + maxScore = score + town = townObj + } + } + if(town){ + town.score = maxScore + } + return town + } + + + def getSimilarTown(regions, fullCity, townstr){ + def town = null + for(def r : regions){ + def townList = commMap.get(fullCity.city+"#"+r) + + for(def townObj : townList){ + if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){ + town = townObj + }else if(townObj.alias && townObj.alias.contains(townstr)){ + town = townObj + }else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){ + def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","") + if(townstr.contains(newTown)){ + town = townObj + } + } + if(town){ + break + } + } + } + return town + } + + + def getFullRegion(fullCity, d){ + def regions = regionMap.get(fullCity.city) + def names = [] + if(regions){ + for(def r : regions){ + if(r.mzb_district_name.contains(d.region)){ + names << r.district + } + } + } + return names; + } + + + def createDatas(dists){ + def rets = [] + rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"] + for(def d : dists){ + rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score] + } + return rets; + } + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy index 7c9472e42..7d34463a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -1,13 +1,46 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection class TradeDatasDistBatch { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com") static main(args) { -// def + def task = new TradeDatasDistBatch() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d-> + + def udata = new BasicDBObject() + udata.put("city", d['A']) + udata.put("region", d['B']) + udata.put("name", d['C']) + udata.put("alias", d['D']) + alias_fang_COLL.save(udata) +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// def flag = null +// if(!dist){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district") +// if(rets && rets.size()==1){ +// flag = "别名匹配" +// dist = rets[0] +// } +// }else{ +// flag = "名称匹配" +// } +// if(flag){ +// d['F'] = dist.city +// d['G'] = dist.region +// d['H'] = dist.name +// } + }) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index 07d8e5679..63f75b256 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch { def stop = false def month = -1 while(!stop){ - if(sDate<202110){ + if(sDate<202004){ def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) println weekend.toString() +" "+sDate runDatas(citys) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index dbbf916d8..f0c4282e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,10 +33,10 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" -// addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" + addGXDPrice(filePath) // addCcbPrice(filePath) - addThirdPrice(filePath) +// addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() @@ -241,12 +241,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['L']) { - def dist = CommonDatautils.findDistrictById(d['E']) + if (!d['M']) { + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['L'] = gxdPrice + d['M'] = gxdPrice } } }) @@ -270,16 +270,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['N']){ - if(d['C'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['C']) + if(!d['L']){ + if(d['B'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ // d['W'] = dbo.avg_price // d['X'] = dbo.cal_type - d['N'] = dbo.avg_price_2b + d['L'] = dbo.avg_price_2b } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index 924db2def..35f821ce3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '上海市奉贤区星火开发区莲塘路251号8幢', + 'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 0682ee62c..63996e918 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -80,13 +80,13 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() - sysnchVillage() +// sysnchVillage() -// addTownship() + addTownship() } def addTownship(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d-> if(d['E']){ // def gaodeMap = getTownshipInfo(d['F']+","+d['E']) // if(gaodeMap && gaodeMap.township){ @@ -114,7 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{ d['L'] = "是" } } - }) + },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 19c0faebf..907e2a572 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -98,6 +98,23 @@ class InsidePolygon { } return list; } + + public static List> createPoint4(def pointList){ + List> alllist = new ArrayList<>(); + for(def pointArr : pointList){ + List list = new ArrayList<>(); + for(def arr : pointArr){ + Point point = new Point(); + double x = arr[0]; + double y = arr[1]; + point.setLon(x); + point.setLat(y); + list.add(point); + } + alllist << list + } + return alllist + } /** * 判定一个经纬度点与区域的关系 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index f14bedadd..6aba12207 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -76,7 +76,6 @@ class ElasticUtils { connection.close() } - static RestHighLevelClient getClient(String host) { RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { @@ -101,6 +100,30 @@ class ElasticUtils { } + static RestHighLevelClient getClient89(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + /** * 创建一个ES批处理器 * @param host diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 7e85c66ad..50882747b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 5 + def nThread = Runtime.getRuntime().availableProcessors() * 4 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index ab5c90233..b3b119384 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995 /** * 经纬度转地址