From 816798f25b3f4375f5b254a2a05fefd6631e4d65 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 15 Nov 2024 10:15:49 +0800 Subject: [PATCH 01/20] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E8=84=9A=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddCityRegionPrice.groovy | 8 +- .../data_handle/AutoAddDistrictBatch.groovy | 54 +++++--- .../gj/data_handle/DistrictRepeatBatch.groovy | 127 +++++++++++++++++- .../gj/data_handle/RegionStandardBatch.groovy | 62 +++++++-- .../gj/data_handle/TownStandardBatch.groovy | 24 ++-- 5 files changed, 230 insertions(+), 45 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index 954e0402b..a42b84aeb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil import com.fangjia.script.averager.InsertMonitorCalcCommandBatch import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.MongoClient -import com.rabbitmq.client.AMQP class AddCityRegionPrice { @@ -24,7 +22,7 @@ class AddCityRegionPrice { static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() - task.addCityprice() +// task.addCityprice() // task.addDistrictprice() // task.addCityRentPrice() // task.addRegionRentPrice() @@ -32,7 +30,7 @@ class AddCityRegionPrice { // task.updateRegionRentData() // task.updateDealPriceAreaType() -// task.checkCityImport() + task.checkCityImport() } def addRegionPrice(){ @@ -255,7 +253,7 @@ class AddCityRegionPrice { def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1) def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1) if(startObj && endObj){ - def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409") + def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410") months.each {m-> def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger()) if(!obj){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index 8e1a842fe..80a9e2bdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - /** * 小区自动入库(新增小区,每月运行一次) */ @@ -37,7 +36,7 @@ class AutoAddDistrictBatch extends BaseBatch{ @Override void process(Object args) { //查找第三方匹配没有匹配上的小区 - def citys = ["上海"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() def list = [] list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] for(def c : citys){ @@ -52,6 +51,18 @@ class AutoAddDistrictBatch extends BaseBatch{ fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx" } ExcelUtils.write(fileName, list, "待入库数据") + + + check(fileName) + } + + def check(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district") + if(dists){ + d['J'] = "库中已存在" + } + }) } def checkDatas(keysets){ @@ -62,8 +73,8 @@ class AutoAddDistrictBatch extends BaseBatch{ def totalCount = new AtomicLong() for(def i=0;i - if(d.name == name && d.type.contains("住宅") && d.shape){ + if(d.name == name && d.type.contains("住宅") ){ //&& d.shape obj = d } }) @@ -154,6 +177,7 @@ class AutoAddDistrictBatch extends BaseBatch{ q.put("match_id", new BasicDBObject('$exists',false)) q.put("old_match_id", new BasicDBObject('$exists',false)) q.put("city", c) + q.put("name", "钱江大厦") def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) def rMap = [:] for(def d : dbcursor){ @@ -228,7 +252,7 @@ class AutoAddDistrictBatch extends BaseBatch{ q.put("district", district) q.put("name", name) q.put('std_tag', '房地产;住宅区') - def count = BAIDU_POI_COLL.count(q) - return count; + def obj = BAIDU_POI_COLL.findOne(q) + return obj; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy index 3fb5a0d48..74d0ddbcb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy @@ -1,17 +1,28 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang.SerializationUtils +import org.bson.types.ObjectId /** * 查找重复小区 */ class DistrictRepeatBatch { + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + static main(args) { def task = new DistrictRepeatBatch() - task.run() +// task.run() + task.resultProcess() } def run(){ @@ -65,4 +76,118 @@ class DistrictRepeatBatch { return rows; } + + def resultProcess(){ + ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d-> + if(d["O"] && !d["O"].contains("不处理") && !d['R']){ + if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){ + def updteId = d['H'] + def q = new BasicDBObject() + q.put("_id", new ObjectId(updteId)) + + def udata = new BasicDBObject() + if(d['O']=="K列改为商住" ){ + udata.put("estate_type2", "商住") + }else{ + udata.put("estate_type2", "别墅") + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + saveOptLog(d["I"], d["J"], d["K"], "update") + d["R"] = "库中已处理" + } + else if(d["O"] == "都删"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata)) + saveOptLog(d["B"], d["C"], d["D"], "remove") + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata)) + saveOptLog(d["I"], d["J"], d["K"], "remove") + d["R"] = "库中已处理" + } + else if(d["O"]=="删除K列小区"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata)) + saveOptLog(d["I"], d["J"], d["K"], "remove") + + def obj = CommonDatautils.findDistrictById(d['A']) + if(d['C']==d['J']){ + def alias = [] + def addr_alias = [] + if(d['D']!=d['K']){ + def s1 = d['D'].replace(d['K'], "") + def s2 = d['K'].replace(d['D'], "") + def addNameAlias = true + if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){ + addNameAlias = false + } + if(addNameAlias){ + + if(obj.alias){ + alias = SerializationUtils.clone(obj.alias) + } + if(!alias.contains(d['K'])){ + alias << d['K'] + } + } + } + + def address = null + if(d['M'] && d['F']!=d['M']){ + def formatAddress = AddressUtil.formatAddress(d['A'], d['M']) + if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){ + if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){ + if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){ + address = formatAddress + }else{ + if(obj.addr_alias){ + addr_alias = SerializationUtils.clone(obj.addr_alias) + } + if(!addr_alias.contains(formatAddress)){ + addr_alias << formatAddress + } + } + } + } + } + + if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){ + def udata2 = new BasicDBObject() + if(alias){ + udata2.put("alias", alias) + } + if(addr_alias){ + udata2.put("addr_alias", addr_alias) + } + if(address){ + udata2.put("address", address) + } + udata2.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2)) + } + } + + d["R"] = "库中已处理" + } + } + }) + } + + def saveOptLog(city, region, name, type){ + def obj = new BasicDBObject(); + obj.put("city",city); + obj.put("region",region); + obj.put("district_name",name); + obj.put("category","district"); + obj.put("c_date", new Date()) + obj.put("m_date", new Date()) + obj.put("type", type) + obj.put("user_name", "高娟") + obj.put("origin", "script") + OPERATION_LOG_COLL.save(obj) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index c78e0a84b..0afba947a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject +import org.apache.commons.io.FileUtils import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions @@ -41,12 +42,11 @@ class RegionStandardBatch extends BaseBatch{ def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] def index = 0 - def citys = ["攀枝花"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() for(def city : citys){ index ++ -// if(index<32){ -// continue -// } + + if(runCitys.contains(city)){ continue } @@ -57,19 +57,32 @@ class RegionStandardBatch extends BaseBatch{ } if(fullCityName){ - def rMap = regionFullNameMap.get(fullCityName) - if(rMap){ - def set = rMap.keySet() - if(set.size()>1){ - def rets = findEsPoiByCity(fullCityName) - println city +" "+rets.size() - poiHaveRegionProcess(city, set, rMap, rets) - }else{ + def fileName = "/home/goojia/gj/poi/${fullCityName}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\poi\\${fullCityName}.csv" + } + File file=new File(fileName); + if(file.exists()){ + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + if(set.size()>1){ +// def rets = findEsPoiByCity(fullCityName) + def rets = readCSV(fileName) + println city +" "+rets.size() + poiHaveRegionProcess(city, set, rMap, rets) + }else{ // haveStreetProcess(city, set, cat) + + println "没找区域:"+city + } + }else{ + println "没找区域:"+city } }else{ - println "没找区域:"+city + println fileName+" 文件没找到" } + }else{ println "没找到相应的全称:"+city } @@ -365,6 +378,29 @@ class RegionStandardBatch extends BaseBatch{ // // // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 // } + def readCSV(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + for(line in lines) { + if(line){ + String[] arrs = line.split("\\|@\\|") + def o = new BasicDBObject() + try{ + o.put("city", arrs[0]) + o.put("district", arrs[1]) + o.put("name", arrs[2]) + o.put("township", arrs[3]) + o.put("status", arrs[4]) + o.put("id", arrs[5]) + rets << o + }catch(Exception e){ + println filename+" 文件异常" + } + } + + } + return rets; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index cbe40d902..545a0d186 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -66,7 +66,7 @@ class TownStandardBatch extends BaseBatch{ def townStandardByPoi(){ updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"] - def citys = CommonDatautils.findAllCity() + def citys = ["福州"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -88,7 +88,7 @@ class TownStandardBatch extends BaseBatch{ def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"]) def set = rMap.keySet() // def rets = findEsPoiByCity(fullCityName) - def rets = readPoiCsv(fullCityName) + def rets = readPoiCsv(fileName) println city +" "+rets.size() if(set.size()>1){ poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap) @@ -96,6 +96,8 @@ class TownStandardBatch extends BaseBatch{ // haveStreetProcess(city, set, cat) } } + }else{ + println fileName+" 文件没找到" } }else{ println "没找到相应的全称:"+city @@ -201,16 +203,16 @@ class TownStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["township":updateTown]) results << m -// if(results.size()%100==0){ -// updateByEs(results) -// results.clear() -// } + if(results.size()%100==0){ + updateByEs(results) + results.clear() + } updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown] println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown }else{ def m = [:] m.put(d.id, ["township":""]) - results << m +// results << m // if(results.size()%100==0){ // updateByEs(results) // results.clear() @@ -224,10 +226,10 @@ class TownStandardBatch extends BaseBatch{ } } -// if(results.size()>0){ -// updateByEs(results) -// results.clear() -// } + if(results.size()>0){ + updateByEs(results) + results.clear() + } println fullCityName +" 处理完成" } From d119dfad55bb134b5f34436a2f5a61d25f77204c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 15 Nov 2024 14:24:45 +0800 Subject: [PATCH 02/20] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=AF=BC=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 95 ++++++++++++++++++- .../gj/data_handle/TempExportData.groovy | 54 ++++++++++- 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index a649c273c..60c80de5a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -2656,12 +2656,12 @@ class CommonDatautils { } - public static def queryDistrictPrices(city, region, name, estateType, startDate, endDate){ + public static def queryDistrictPrices(city, region, name, startDate, endDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); queryCondition.put("region", region); queryCondition.put("district_name", name); - queryCondition.put("estate_type", estateType); +// queryCondition.put("estate_type", estateType); queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate))); DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) def prices = []; @@ -4107,6 +4107,46 @@ class CommonDatautils { } + + static def findHouseCountByDate(def city, def region, def name, def alias, def cat, def startDate){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("region", region)) + .must(QueryBuilders.termQuery("cat", cat)).must(date_range) + + if(name || alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + } + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + return houses.size(); + } + + /** * 是否是别墅 * @return @@ -5259,4 +5299,55 @@ class CommonDatautils { return STATS_INDEX_COLL.findOne(q) } + + static def findAllThirdPrice(city, region, name, statDate){ + def query = new BasicDBObject() + query.append("category", "district") + query.append("city", city) + query.append("region", region) + query.append("name", name) + query.append("s_date", new BasicDBObject(QueryOperators.GTE, statDate)) + def dbCursor = thirdpartColl.find(query).sort(new BasicDBObject("s_date",-1)) + def retMap = [:] + for(def d : dbCursor){ + + if(!retMap.get("xingfuli") && d.get("xingfuli_esf_price")){ + retMap.put("xingfuli", d.get("xingfuli_esf_price")) + } + if(!retMap.get("anjuke") && d.get("anjuke_esf_price")){ + retMap.put("anjuke", d.get("anjuke_esf_price")) + } + + if(!retMap.get("soufang") && d.get("soufang_esf_price")){ + retMap.put("soufang", d.get("soufang_esf_price")) + } + if(!retMap.get("lianjia") && d.get("lianjia_esf_price")){ + retMap.put("lianjia", d.get("lianjia_esf_price")) + } + if(!retMap.get("lianjia") && d.get("beike_esf_price")){ + retMap.put("lianjia", d.get("beike_esf_price")) + } + + if(!retMap.get("cityhouse") && d.get("cityhouse_esf_price")){ + retMap.put("cityhouse", d.get("cityhouse_esf_price")) + } + if(!retMap.get("jiwu") && d.get("jiwu_esf_price")){//使用吉屋替换房探 待抓取 + retMap.put("jiwu", d.get("jiwu_esf_price")) + } + + if(!retMap.get("house365") && d.get("365house_esf_price")){//使用淘房365替换房多多 + retMap.put("house365", d.get("365house_esf_price")) + } + + if(!retMap.get("maitian") && d.get("maitian_esf_price")){ + retMap.put("maitian", d.get("maitian_esf_price")) + } + if(!retMap.get("tc_58") && d.get("58tc_esf_price")){ + retMap.put("tc_58", d.get("58tc_esf_price")) + } + } + + return retMap; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy index 4d865d728..f6ac01ab0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy @@ -1,15 +1,14 @@ package com.fangjia.script.gj.data_handle -import com.fangjia.core.RegionData -import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.ExcelUtils +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong class TempExportData { @@ -17,7 +16,10 @@ class TempExportData { static main(args) { def task = new TempExportData() - task.run() +// task.run() + def rets = task.fzHouseStat() + ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据") + println "文件已生成" } def run() { @@ -75,4 +77,48 @@ class TempExportData { println(city +" "+rets.size()) return rets; } + + def fzHouseStat(){ + def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "alias","estate_type2"]) + districts = districts[0..10] + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def total = districts.size() + def rets = new CopyOnWriteArrayList() + rets << ["城市","区域","名称","上月价格","3年房源量","2年房源量", "2年第三方家数"] + for(def i=0;i Date: Fri, 15 Nov 2024 14:48:44 +0800 Subject: [PATCH 03/20] =?UTF-8?q?=E5=8E=BB=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/TempExportData.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy index f6ac01ab0..d9d86dbf8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy @@ -80,7 +80,6 @@ class TempExportData { def fzHouseStat(){ def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "alias","estate_type2"]) - districts = districts[0..10] def addCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) def total = districts.size() @@ -88,8 +87,8 @@ class TempExportData { rets << ["城市","区域","名称","上月价格","3年房源量","2年房源量", "2年第三方家数"] for(def i=0;i Date: Tue, 10 Dec 2024 17:48:23 +0800 Subject: [PATCH 04/20] =?UTF-8?q?=E6=B6=89=E5=8F=B0=E5=9B=AD=E5=8C=BA?= =?UTF-8?q?=E6=94=B9=E5=9B=9E=E5=8E=9F=E6=9D=A5=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy index 4efd79e7f..25beca0a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy @@ -18,7 +18,7 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class RelatedTaiWanBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM")); DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020) def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"] @@ -40,9 +40,10 @@ class RelatedTaiWanBatch extends BaseBatch{ //createDatas() //企业库获取涉台园区企业 updateRangeByPoint() //更改企业所在自贸区 export() //导出数据 - }else{ - exportEmptyFile() } +// else{ +// exportEmptyFile() +// } } From 8fab75476a6fc739c497cf7b280fd2cb33d2295a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 19 Nov 2024 18:56:39 +0800 Subject: [PATCH 05/20] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 108 ++++++++++++++++++ .../data_handle/AutoAddDistrictBatch.groovy | 8 +- 2 files changed, 112 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 60c80de5a..43251bcc5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4147,6 +4147,96 @@ class CommonDatautils { } + static def findHousePriceByDate(def city, def region, def name, def alias, def cat, def startDate){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("region", region)) + .must(QueryBuilders.termQuery("cat", cat)).must(date_range) + + if(name || alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + } + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + def prices = [] + if(houses){ + for (int k=0;k + index++ + println index + rets << [map.cityname, map.adname, map.name, map.address, map.location.x, map.location.y] + }) + return rets; + } + + + static def findOneThirdNetWorkOffice(city, region, name){ + String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'" + def map = [:] + + ElasticUtils.executeSqlQuery(sql, { m-> + if(m.location && !map){ + map = ["lng":m.location.x, "lat":m.location.y] + } + }) + return map + } + + + static def findThirdNetWorkOffice(){ + String sql = "select city,region,name,address,location from office where city='烟台' or city='北京'" + def rets = [] + def index= 0 + ElasticUtils.executeSqlQuery(sql, { map-> + index++ + println index + if(map.location){ + rets << [map.city, map.region, map.name, map.address, map.location.lon, map.location.lat] + }else{ + rets << [map.city, map.region, map.name, map.address, "", ""] + } + + }) + return rets; + } /** * 是否是别墅 * @return @@ -5350,4 +5440,22 @@ class CommonDatautils { return retMap; } + static getSeaweedArroundByCat(city, lng, lat, cat) { + try { + def query = new BasicDBObject() + query.append("cat", cat) + query.append("city", city) + query.append("visible", 0) + DBObject sdq = new BasicDBObject(); + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0)); + query.append('$sd', sdq) + return DbUtils.seaweed.find(query).toList() + } catch (Exception e) { + e.printStackTrace() + } + return 0 + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index 80a9e2bdc..ae2703c0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -73,8 +73,8 @@ class AutoAddDistrictBatch extends BaseBatch{ def totalCount = new AtomicLong() for(def i=0;i Date: Wed, 20 Nov 2024 13:55:26 +0800 Subject: [PATCH 06/20] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 ++ .../data_handle/AutoAddDistrictBatch.groovy | 1 - .../data_handle/ThirdPriceMappingBatch.groovy | 6 +- .../script/gj/xiamen/CryptoSm4Util.groovy | 115 ++++++++++++++++++ .../gj/xiamen/RelatedTaiWanBatch.groovy | 27 ++-- 5 files changed, 143 insertions(+), 13 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index b98b25ab1..e639f76e3 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -530,6 +530,13 @@ + + + org.bouncycastle + bcpkix-jdk15to18 + 1.70 + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index ae2703c0a..5367882e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -177,7 +177,6 @@ class AutoAddDistrictBatch extends BaseBatch{ q.put("match_id", new BasicDBObject('$exists',false)) q.put("old_match_id", new BasicDBObject('$exists',false)) q.put("city", c) - q.put("name", "钱江大厦") def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) def rMap = [:] for(def d : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 329a548fb..9af4b1ec4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -4,7 +4,6 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil @@ -12,14 +11,11 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.bson.types.ObjectId -import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - /** * 第三方小区与我们小区映射关系表 */ @@ -44,7 +40,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { // //将新增数据同步过来 -// addNemCommunity() + addNemCommunity() // //接口进行匹配 match() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy new file mode 100644 index 000000000..6e960180d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy @@ -0,0 +1,115 @@ +package com.fangjia.script.gj.xiamen + +import org.bouncycastle.crypto.engines.SM4Engine; +import org.bouncycastle.crypto.modes.CBCBlockCipher; +import org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher; +import org.bouncycastle.crypto.params.KeyParameter; +import org.bouncycastle.crypto.params.ParametersWithIV; +import org.bouncycastle.util.encoders.Hex; + +import java.security.SecureRandom; + +/** + * @author shenjian + * @since 2024/10/9 + */ +public class CryptoSm4Util { + + /** + * 默认密钥 + */ + private static final String DEFAULT_SECRET_KEY = "A1B2C3D4E5F60789A1B2C3D4E5F60789"; + private static final String CHARSET = "UTF-8"; + + // String plaintext -> Base64-encoded String ciphertext + public static String encrypt(String key, String plaintext) { + try { + byte[] keyBytes = Hex.decode(key); + byte[] iv = generateIV(); + PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine())); + cipher.init(true, new ParametersWithIV(new KeyParameter(keyBytes), iv)); + byte[] input = plaintext.getBytes(CHARSET); + byte[] encrypted = new byte[cipher.getOutputSize(input.length)]; + int length1 = cipher.processBytes(input, 0, input.length, encrypted, 0); + int length2 = cipher.doFinal(encrypted, length1); + byte[] finalEncrypted = new byte[length1 + length2]; + System.arraycopy(encrypted, 0, finalEncrypted, 0, finalEncrypted.length); + return Hex.toHexString(iv) + Hex.toHexString(finalEncrypted); + } catch (Exception ex) { + ex.printStackTrace(); + } + return null; + } + + private static byte[] generateIV() { + byte[] iv = new byte[16]; + new SecureRandom().nextBytes(iv); + return iv; + } + + // Base64-encoded String ciphertext -> String plaintext + public static String decrypt(String key, String ciphertext) { + try { + byte[] keyBytes = Hex.decode(key); + byte[] iv = Hex.decode(ciphertext.substring(0, 32)); + byte[] encryptedData = Hex.decode(ciphertext.substring(32)); + PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine())); + cipher.init(false, new ParametersWithIV(new KeyParameter(keyBytes), iv)); + byte[] decrypted = new byte[cipher.getOutputSize(encryptedData.length)]; + int length1 = cipher.processBytes(encryptedData, 0, encryptedData.length, decrypted, 0); + int length2 = cipher.doFinal(decrypted, length1); + byte[] finalDecrypted = new byte[length1 + length2]; + System.arraycopy(decrypted, 0, finalDecrypted, 0, finalDecrypted.length); + return new String(finalDecrypted, CHARSET).trim(); + } catch (Exception ex) { + ex.printStackTrace(); + } + return null; + } + + /** + * 使用默认密钥加密 + * + * @param str 加密内容 + * @return 以String密文输入, String明文输出 + */ + public static String encode(String str) { + return encode(DEFAULT_SECRET_KEY, str); + } + + /** + * 自定义密钥加密 + * + * @param key 自定义密钥 + * @param str 加密的内容 + * @return String明文输入, String密文输出 + */ + public static String encode(String key, String str) { + return encrypt(key, str); + } + + /** + * 使用默认密钥解密 + * + * @param str 解密内容 + * @return 以String密文输入, String明文输出 + */ + public static String decode(String str) { + return decode(DEFAULT_SECRET_KEY, str); + } + + /** + * 自定义密钥解密 + * + * @param key 自定义密钥 + * @param str 加密内容 + * @return 以String密文输入, String明文输出 + */ + public static String decode(String key, String str) { + return decrypt(key, str); + } + + public static void main(def a){ + println encrypt( DEFAULT_SECRET_KEY, "测试") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy index 25beca0a7..e44d4424e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy @@ -15,9 +15,13 @@ import com.mongodb.DBCollection /** * 厦门涉台园区数据同步 */ + + @BatchOwner(Watcher.GaoJuan) class RelatedTaiWanBatch extends BaseBatch{ + + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM")); DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020) @@ -28,6 +32,8 @@ class RelatedTaiWanBatch extends BaseBatch{ def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"] + + static main(args) { runBatch(new RelatedTaiWanBatch(), args); } @@ -37,14 +43,10 @@ class RelatedTaiWanBatch extends BaseBatch{ def day = new Date().getDate() if(day == 10){ - //createDatas() //企业库获取涉台园区企业 +// createDatas() //企业库获取涉台园区企业 updateRangeByPoint() //更改企业所在自贸区 export() //导出数据 } -// else{ -// exportEmptyFile() -// } - } @@ -153,10 +155,18 @@ class RelatedTaiWanBatch extends BaseBatch{ } def allDatas = [] allDatas.addAll(rows) -// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false); - FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|"); + + //原不加密之前 +// FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|"); +// XmBankFtpUtils.ftpUpload(fileName, curDate) + + + def str1 = FileUtils.createFileStringByCustomSuffix(rows, fileName, "utf-8", "|@|","|+|"); + def str1_encryptedText = CryptoSm4Util.encrypt( CryptoSm4Util.DEFAULT_SECRET_KEY, str1) + FileUtils.createEncryptFile(fileName, str1_encryptedText, false) XmBankFtpUtils.ftpUpload(fileName, curDate) + def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy" if(CommonDatautils.isWindow()){ fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy" @@ -182,4 +192,7 @@ class RelatedTaiWanBatch extends BaseBatch{ FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); XmBankFtpUtils.ftpUpload(fileName2, curDate) } + + + } From b0f42e88688a7be498817f75b3f5d3f1f635b547 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Nov 2024 11:15:56 +0800 Subject: [PATCH 07/20] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/ThirdPriceMappingBatch.groovy | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 9af4b1ec4..5acefdb73 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -40,9 +40,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { // //将新增数据同步过来 - addNemCommunity() -// //接口进行匹配 - match() +// addNemCommunity() +//// //接口进行匹配 +// match() addOldMatchResult() } @@ -177,6 +177,10 @@ class ThirdPriceMappingBatch extends BaseBatch{ def citys = CommonUtils.findAllCity() def keys = new BasicDBObject() keys.put("_id", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("name", 1) + keys.put("source", 1) for(def c : citys){ def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList() @@ -196,8 +200,13 @@ class ThirdPriceMappingBatch extends BaseBatch{ void run() { try{ addCount.addAndGet(1) - def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString()) - def obj = dataProcessColl.findOne(query) + //.append("s_date", curDate) + def query = new BasicDBObject().append("city", city).append("region",d.district).append("name", d.name).append("source",d.source) + def objs = dataProcessColl.find(query).sort(new BasicDBObject("s_date",-1)).limit(1) + def obj = null + if(objs){ + obj = objs[0] + } if(obj) { def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district") def udata = new BasicDBObject() From 6a6549756c6fab7216eda85a3f8880e0e04ccc67 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Nov 2024 15:30:31 +0800 Subject: [PATCH 08/20] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/ThirdPriceMappingBatch.groovy | 7 +++- .../thirdPriceCrawl/DataStandardBatch.groovy | 37 ++++++------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 5acefdb73..a6180d973 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -42,14 +42,19 @@ class ThirdPriceMappingBatch extends BaseBatch{ // //将新增数据同步过来 // addNemCommunity() //// //接口进行匹配 -// match() + match() addOldMatchResult() } def match(){ def citys = CommonUtils.findAllCity() + def index = 0 for(def c : citys){ + index ++ + if(index < 206){ + continue + } def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false)) def noMatchDatas = thirdSeaweedMappingColl.find(query).toList() matchByCity(c, noMatchDatas) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 25e01d8a5..b57cdd659 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -7,21 +7,8 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.NameProcess import com.fangjia.script.gj.district.SeaweedDataMatchProcess -import com.fangjia.script.linxsh.util.CommonUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.CosineSimilarAlgorithm -import com.fangjia.script.util.DataUtil -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DB -import com.mongodb.DBCollection -import com.mongodb.DBObject -import com.mongodb.QueryOperators -import net.sf.json.JSONObject +import com.fangjia.script.util.* +import com.mongodb.* import org.apache.commons.lang3.SerializationUtils import org.apache.commons.lang3.StringUtils import org.apache.tools.ant.util.DateUtils @@ -31,9 +18,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger -import java.util.regex.Matcher import java.util.regex.Pattern - /** * 第三方数据标准化处理 * 成功type说明 @@ -689,14 +674,15 @@ class DataStandardBatch extends BaseBatch{ def pre = (data.avg_price-price)*100/price if(pre>=-20 && pre<=20){ return dbo; - }else{ - def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) - if(!ret){ - data.put("msg","房源价格过滤") - THIRD_PRICE_FILTER_COLL.save(data) - } - return null } +// else{ +// def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) +// if(!ret){ +// data.put("msg","房源价格过滤") +// THIRD_PRICE_FILTER_COLL.save(data) +// } +// return null +// } } } return dbo @@ -934,7 +920,8 @@ class DataStandardBatch extends BaseBatch{ DBObject qdata = new BasicDBObject(); qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE)) // qdata.put("city","上海") -// qdata.put("name","白金瀚宫") +// qdata.put("name","绿地云天坊") +// qdata.put("status", 3) // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); From ec7d9c8dc5e35961d06447ebb51b78c0b2cfc83c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Nov 2024 15:49:33 +0800 Subject: [PATCH 09/20] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E5=B0=8F=E5=8C=BA=E8=A6=86=E7=9B=96=E7=8E=87=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/ThardParthAnalyBatch.groovy | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index 6a2ee9eec..8127ca79c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -1,6 +1,6 @@ package com.fangjia.script.gj.data_handle -import com.fangjia.datastore.driver.FacetResult + import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -10,17 +10,15 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils -import com.mongodb.AggregationOutput import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.mongodb.DBObject @BatchOwner(Watcher.GaoJuan) class ThardParthAnalyBatch extends BaseBatch{ def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com") - + static DBCollection data_process_coll = MongoUtils.getCollectionByName("fangjia_stat", "data_process", "mongo.db.fangjia.com") static main(args) { runBatch(new ThardParthAnalyBatch(), args) @@ -119,11 +117,14 @@ class ThardParthAnalyBatch extends BaseBatch{ } def findMatchSuccessDistCount(sDate){ - def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4) - def dbcursor = thirdPartyColl.find(q) +// def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4) +// def dbcursor = thirdPartyColl.find(q) + + def q = new BasicDBObject().append("s_date", sDate) + def dbcursor = data_process_coll.find(q) def set = new HashSet() for (def d : dbcursor) { - set << d.city+"##"+d.region+"##"+d.name + set << d.match_city+"##"+d.match_region+"##"+d.match_name } return set.size() } From 0b9d2cdfda8e45a590835ffdc1012f1308fa7863 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Nov 2024 18:01:18 +0800 Subject: [PATCH 10/20] update --- .../fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index 8127ca79c..6d9260f87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -18,7 +18,7 @@ class ThardParthAnalyBatch extends BaseBatch{ def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com") - static DBCollection data_process_coll = MongoUtils.getCollectionByName("fangjia_stat", "data_process", "mongo.db.fangjia.com") + static DBCollection data_process_coll = CrawalDBUtil.getDataProcessCol() static main(args) { runBatch(new ThardParthAnalyBatch(), args) From 1ad116d67354827718a512b95157ea53107e3708 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Nov 2024 13:39:06 +0800 Subject: [PATCH 11/20] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 14 ++++------ .../script/commData/CommonDatautils.groovy | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index f75cbb9f1..c65f09175 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -10,12 +10,10 @@ import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql import org.bson.types.ObjectId @@ -23,11 +21,9 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong import java.util.regex.Pattern - /** * 建行模型估值项目 * 成交数据同步 @@ -71,11 +67,11 @@ class BigdataDealBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) println "curDate:"+curDate -// def allCitys = CommonDatautils.findAllCity() -// for(def city : allCitys){ -// addDealPriceMatch(city); -// } -// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ + addDealPriceMatch(city); + } + updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 def rows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 43251bcc5..4d1ea8f76 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4208,6 +4208,19 @@ class CommonDatautils { } + static def findAllNewPoiDatas(city){ + String sql = "select cityname,adname,name,address,location from poi_amap_2024 where cityname='${city}'" + def rets = [:] + def index= 0 + ElasticUtils.executeSqlQuery(sql, { map-> + index++ + println index + rets << ["map_id":map.id,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "type":map.type,"lng":map.location.x, "lat":map.location.y] + }) + return rets; + } + + static def findOneThirdNetWorkOffice(city, region, name){ String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'" def map = [:] @@ -4903,6 +4916,21 @@ class CommonDatautils { return cityMap } + static def findFullNameCitys(){ + def q = new BasicDBObject() + q.put("type","城市") + + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("province", 1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys) + def cityList = [] + for(def d : dbcursor){ + cityList << d + } + return cityList + } + static def queryDistNameByCity(){ def q = new BasicDBObject() From 502c682c8bdf8bd9f45668a2dad08afd8ca14a03 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Nov 2024 18:34:02 +0800 Subject: [PATCH 12/20] =?UTF-8?q?=E6=96=B0=E7=89=88poi=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 34 ++- .../gj/data_handle/NewGaoDePoiBatch.groovy | 232 ++++++++++++++++++ 2 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 4d1ea8f76..a1bc0bd49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4209,13 +4209,13 @@ class CommonDatautils { static def findAllNewPoiDatas(city){ - String sql = "select cityname,adname,name,address,location from poi_amap_2024 where cityname='${city}'" - def rets = [:] + String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' limit 100" + def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> index++ println index - rets << ["map_id":map.id,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "type":map.type,"lng":map.location.x, "lat":map.location.y] + rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] }) return rets; } @@ -5149,6 +5149,34 @@ class CommonDatautils { } + static def queryFullRegionTownByCity(city, fields){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + if(d.district){ + def list = townMap.get(d.district) + if(!list){ + list = [] + } + list << d + townMap.put(d.district, list) + } + } + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } + + + static def queryTownByCity(city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy new file mode 100644 index 000000000..b8621d41c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -0,0 +1,232 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ExcelUtils +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +/** + * 新版poi处理入库(第二批) + */ +@BatchOwner(Watcher.GaoJuan) +class NewGaoDePoiBatch extends BaseBatch{ + + public static def version = new Date().getTime() + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + public static def TAGS = [] + + static main(args) { + runBatch(new NewGaoDePoiBatch(), args) + } + + @Override + void process(Object args) { + run() + } + + def run(){ + if (CommonDatautils.isWindow()) { + TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx") + }else{ + TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx") + } + + + def cityList = CommonDatautils.findFullNameCitys().reverse() + for(def cityObj : cityList){ + if(cityObj.city == "上海市"){ + def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) + println cityObj.city+" "+results.size() + def rows = running(cityObj.city, results) + save(rows) + } + } + } + def getAllPoiType(filePath){ + def poiTypes = ExcelUtils.read(filePath, 0, 1) + def types = [] + for(def poi : poiTypes){ + types << poi['A'] + } + return types + } + + def running(city,results){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"]) + for(def i=0;i=1){ + saveObj.type = arrs[0] + } + if(arrs.size()>=2){ + saveObj.secondary_type = arrs[1] + }else{ + saveObj.secondary_type = saveObj.type + } + if(arrs.size()>=3){ + def smallstr = arrs[2] + def smallArr = null + if(smallstr.contains("|")){ + smallArr = smallstr.split("|") + } + if(smallArr && smallArr.size()>=1){ + saveObj.small_type = smallArr[0] + }else{ + saveObj.small_type = smallstr + } + }else{ + saveObj.small_type = saveObj.secondary_type + } + + saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type + + } + def location = [:] + location.lon = DataHandleUtils.getDoubleRound(6, map.lng) + location.lat = DataHandleUtils.getDoubleRound(6, map.lat) + saveObj.location = location + saveObj.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ") + saveObj.version = version + saveObj.data_id = map.map_id + saveObj.data_source = "高德地图" + saveObj.data_source_year = "2024-11" + + if(saveObj.cat && TAGS.contains(saveObj.cat)){ + saveObj.status = 0 + }else{ + saveObj.status = 1 + } + + if(location){ + def towns = townMap.get(saveObj.district) + if(towns){ + for(def town : towns){ + if(town.shape && town.shape.coordinates){ + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + saveObj.township = town.street + break + } + } + } + if(!saveObj.township && map.address.contains("街道")){ + println map.address +" 含街道" + } + } + } + + if(map.address){ + if(map.address.length()<3){ + saveObj.township = "" + }else{ + def address = map.address + def i = 0 + while(i<5){ + if(address.toString().startsWith(map.province)){ + address = address.toString().substring(0, map.province.toString().length()) + } + if(address.toString().startsWith(map.city)){ + address = address.toString().substring(0, map.city.toString().length()) + } + if(address.toString().startsWith(map.district)){ + address = address.toString().substring(0, map.district.toString().length()) + } + if(address.toString().startsWith("市辖区")){ + address = address.toString().substring(0, 3) + } + if(address.toString().startsWith(saveObj.township)){ + address = address.toString().substring(0, saveObj.township.toString().length()) + } + } + saveObj.township = address + } + } + return saveObj + } + + def save(rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 1000; + int processedCount = 0; + def client = ElasticUtils89.getClient("192.168.0.89") + + rows.each{key,map-> +// def map = rows.get(key) + + request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON)); + + // 批量提交数据 + if (request.numberOfActions() >= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss") + } + + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + } + +} From d4ed18b6a6d5b2bfebca638b626482194a27aed3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Nov 2024 18:39:29 +0800 Subject: [PATCH 13/20] =?UTF-8?q?bug=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/commData/CommonDatautils.groovy | 2 +- .../script/gj/data_handle/NewGaoDePoiBatch.groovy | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index a1bc0bd49..033959ead 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4209,7 +4209,7 @@ class CommonDatautils { static def findAllNewPoiDatas(city){ - String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' limit 100" + String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' " def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index b8621d41c..8fc228e3d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -74,8 +74,8 @@ class NewGaoDePoiBatch extends BaseBatch{ def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"]) for(def i=0;i Date: Tue, 26 Nov 2024 14:44:55 +0800 Subject: [PATCH 14/20] update --- .../script/commData/CommonDatautils.groovy | 2 +- .../gj/data_handle/NewGaoDePoiBatch.groovy | 35 +++++++++++----- .../gj/data_handle/RegionStandardBatch.groovy | 1 - .../gj/data_handle/TownStandardBatch.groovy | 42 +++++++++++++------ 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 033959ead..c3d473a3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4209,7 +4209,7 @@ class CommonDatautils { static def findAllNewPoiDatas(city){ - String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' " + String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'" def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 8fc228e3d..bd1f5467a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -71,7 +71,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(results.size()) - def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"]) + def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"]) for(def i=0;i + rows.each{map-> // def map = rows.get(key) - request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON)); + request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON)); // 批量提交数据 if (request.numberOfActions() >= batchSize) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 0afba947a..abce57fe0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -46,7 +46,6 @@ class RegionStandardBatch extends BaseBatch{ for(def city : citys){ index ++ - if(runCitys.contains(city)){ continue } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 545a0d186..4d3e00e32 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -24,6 +24,7 @@ class TownStandardBatch extends BaseBatch{ def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() def updateList = [] + def addAliasList = [] static main(args) { runBatch(new TownStandardBatch(), args) @@ -66,7 +67,10 @@ class TownStandardBatch extends BaseBatch{ def townStandardByPoi(){ updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"] - def citys = ["福州"]//CommonDatautils.findAllCity() + + addAliasList << ["城市","区域","乡镇街道", "新增别名"] + //浙江,江苏,安徽 + def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -78,7 +82,7 @@ class TownStandardBatch extends BaseBatch{ def fileName = "/home/goojia/gj/poi/${fullCityName}.csv" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\${fullCityName}.csv" + fileName = "D:\\数据\\poi\\${fullCityName}.csv" } File file=new File(fileName); if(file.exists()){ @@ -104,6 +108,8 @@ class TownStandardBatch extends BaseBatch{ } } ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据") + ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据") + } def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ def results = [] @@ -196,6 +202,18 @@ class TownStandardBatch extends BaseBatch{ } break } +// else{ +// if(town.street.contains(d.township.replace("镇","").replace("街道","").replace("乡",""))){ +// updateTown = town.street +// opt = false +// def addObj = [town.city, town.district, town.street, d.township] +// if(!addAliasList.contains(addObj)){ +// addAliasList << addObj +// } +// +// break +// } +// } } } if(opt){ @@ -203,12 +221,12 @@ class TownStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["township":updateTown]) results << m - if(results.size()%100==0){ - updateByEs(results) - results.clear() - } +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown] - println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown +// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown }else{ def m = [:] m.put(d.id, ["township":""]) @@ -226,10 +244,10 @@ class TownStandardBatch extends BaseBatch{ } } - if(results.size()>0){ - updateByEs(results) - results.clear() - } +// if(results.size()>0){ +// updateByEs(results) +// results.clear() +// } println fullCityName +" 处理完成" } @@ -258,7 +276,7 @@ class TownStandardBatch extends BaseBatch{ println("批量更新中有失败的操作:") bulkResponse.buildFailureMessage().each { println(it) } } else { - println("批量更新成功!") +// println("批量更新成功!") } } catch (Exception e) { e.printStackTrace() From 8ebfab325794fef9ebd2446f52a6b0614930169e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Nov 2024 15:09:02 +0800 Subject: [PATCH 15/20] update --- .../script/gj/data_handle/NewGaoDePoiBatch.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index bd1f5467a..84a7a4c46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -93,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def createData(map, townMap){ def saveObj = [:] - def towns = townMap.get(saveObj.district) + def towns = townMap.get(map.district) saveObj.province = map.province saveObj.city = map.city if(towns){ @@ -158,7 +158,7 @@ class NewGaoDePoiBatch extends BaseBatch{ } } } - if(map.address && saveObj.township){ + if(map.address && !saveObj.township){ for(def town : towns){ if(map.address.contains(town.street)){ saveObj.township = town.street @@ -196,7 +196,12 @@ class NewGaoDePoiBatch extends BaseBatch{ } i++ } - saveObj.address = address + if(address && address.startsWith("(") && address.endsWith(")")){ + saveObj.address = address.substring(1, address.length()-1) + }else{ + saveObj.address = address + } + } } return saveObj From 435bd679bc4f806caa2d8a498b7c7e60f1667786 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Nov 2024 15:20:15 +0800 Subject: [PATCH 16/20] =?UTF-8?q?=E8=BE=93=E5=87=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 84a7a4c46..937f4cbd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -165,7 +165,7 @@ class NewGaoDePoiBatch extends BaseBatch{ break } } - if(map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){ + if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){ println map.address +" 含街道,镇,乡" } } @@ -202,6 +202,7 @@ class NewGaoDePoiBatch extends BaseBatch{ saveObj.address = address } + } } return saveObj From 6b0c7c4b1844e8e64ee7e897364701a725ead373 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Nov 2024 19:01:05 +0800 Subject: [PATCH 17/20] update --- .../script/commData/CommonDatautils.groovy | 8 +- .../gj/data_handle/NewGaoDePoiBatch.groovy | 134 ++++++++++++++---- 2 files changed, 109 insertions(+), 33 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index c3d473a3b..ed3314d22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4213,9 +4213,11 @@ class CommonDatautils { def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> - index++ - println index - rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] +// if(map.address && map.address.contains("五角场")){ + index++ + println index + rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] +// } }) return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 937f4cbd0..15b8005d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -29,6 +29,8 @@ class NewGaoDePoiBatch extends BaseBatch{ public static def version = new Date().getTime() public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + public static def TAGS = [] static main(args) { @@ -50,12 +52,12 @@ class NewGaoDePoiBatch extends BaseBatch{ def cityList = CommonDatautils.findFullNameCitys().reverse() for(def cityObj : cityList){ - if(cityObj.city == "上海市"){ +// if(cityObj.city == "上海市"){ def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) println cityObj.city+" "+results.size() def rows = running(cityObj.city, results) save(rows) - } +// } } } def getAllPoiType(filePath){ @@ -72,13 +74,14 @@ class NewGaoDePoiBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(results.size()) def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"]) + def mzbRegionMap = regionFullNameMap.get(city) for(def i=0;i=2){ +// alias.sort(){ a, b -> +// b.length().compareTo(a.length()); +// } +// } + + for(def regionName : alias){ + if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ + address = address.substring(regionName.toString().length()) + break + } + } + } } - if(address.toString().startsWith("市辖区")){ - address = address.toString().substring(0, 3) + + + if(address && address.toString().startsWith("市辖区")){ + address = address.toString().substring(3) } - if(saveObj.township && address.toString().startsWith(saveObj.township)){ - address = address.toString().substring(0, saveObj.township.toString().length()) + + if(address && towns){ + for(def town : towns){ + def townAlias = town.alias + if(!townAlias){ + townAlias = [] + } + if(!townAlias.contains(town.street)){ + townAlias << town.street + } +// if(townAlias.size()>=2){ +// townAlias = townAlias.sort(){ a, b -> +// b.length().compareTo(a.length()); +// } +// } + + for(def townName : townAlias){ + if(address.startsWith(townName)){ + address = address.substring(townName.toString().length()) +// if(!saveObj.township){ +// saveObj.township = town.street +// } + break + } + } + } } i++ } @@ -201,8 +277,6 @@ class NewGaoDePoiBatch extends BaseBatch{ }else{ saveObj.address = address } - - } } return saveObj From ce66ee421302d0f598e4013183ae0c9530862783 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 27 Nov 2024 18:22:14 +0800 Subject: [PATCH 18/20] update --- .../script/commData/CommonDatautils.groovy | 1 - .../gj/data_handle/NewGaoDePoiBatch.groovy | 223 ++++++++++-------- 2 files changed, 123 insertions(+), 101 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ed3314d22..9db36e3d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4215,7 +4215,6 @@ class CommonDatautils { ElasticUtils.executeSqlQuery(sql, { map-> // if(map.address && map.address.contains("五角场")){ index++ - println index rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 15b8005d6..822eee441 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{ def cityList = CommonDatautils.findFullNameCitys().reverse() for(def cityObj : cityList){ -// if(cityObj.city == "上海市"){ +// if(cityObj.city == "胡杨河市"){ def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) - println cityObj.city+" "+results.size() - def rows = running(cityObj.city, results) - save(rows) + println cityObj.province+" " +cityObj.city+" "+results.size() + if(results){ + def rows = running(cityObj.city, results) + save(rows) + } // } } } @@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{ }); } latch.await() - return rets; + return rets } def getStandardRegion(mzbRegionMap, district){ @@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{ } saveObj.name = map.name + if(map.cat){ - def arrs = map.cat.split(";|:") - if(arrs.size()>=1){ - saveObj.type = arrs[0] - } - if(arrs.size()>=2){ - saveObj.secondary_type = arrs[1] - }else{ - saveObj.secondary_type = saveObj.type - } - if(arrs.size()>=3){ - def smallstr = arrs[2] - def smallArr = null - if(smallstr.contains("|")){ - smallArr = smallstr.split("|") - } - if(smallArr && smallArr.size()>=1){ - saveObj.small_type = smallArr[0] - }else{ - saveObj.small_type = smallstr - } - }else{ - saveObj.small_type = saveObj.secondary_type - } - - saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type - + saveObj = catWash(saveObj, map) } def location = [:] location.lon = DataHandleUtils.getDoubleRound(6, map.lng) @@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{ def towns = townMap.get(saveObj.district) if(location){ - if(towns){ - for(def town : towns){ - if(town.shape && town.shape.coordinates){ - def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) - if(flag){ - saveObj.township = town.street - break - } + saveObj = addTown(saveObj, towns, location) + } + + if(map.address){ + saveObj = addressWash(saveObj, map, mzbRegionMap, towns) + } + return saveObj + } + + def catWash(saveObj, map){ + def arrs = map.cat.split(";|:") + if(arrs.size()>=1){ + saveObj.type = arrs[0] + } + if(arrs.size()>=2){ + saveObj.secondary_type = arrs[1] + }else{ + saveObj.secondary_type = saveObj.type + } + if(arrs.size()>=3){ + def smallstr = arrs[2] + def smallArr = null + if(smallstr.contains("|")){ + smallArr = smallstr.split("|") + } + if(smallArr && smallArr.size()>=1){ + saveObj.small_type = smallArr[0] + }else{ + saveObj.small_type = smallstr + } + }else{ + saveObj.small_type = saveObj.secondary_type + } + + saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type + return saveObj + } + + def addTown(saveObj, towns, location){ + if(towns){ + for(def town : towns){ + if(town.shape && town.shape.coordinates){ + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + saveObj.township = town.street + break } } + } // if(map.address && !saveObj.township){ // for(def town : towns){ // if(map.address.contains(town.street)){ @@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{ // println map.address +" 含街道,镇,乡" // } // } - } } + return saveObj + } - if(map.address){ - if(map.address.length()<3){ - saveObj.address = "" - }else{ - def address = map.address - def i = 0 - while(i<5){ - if(address && address.toString().startsWith(map.province)){ - address = address.toString().substring(map.province.toString().length()) - } - if(address && address.toString().startsWith(map.city)){ - address = address.toString().substring(map.city.toString().length()) - } + def addressWash(saveObj, map, mzbRegionMap, towns){ + if(map.address.length()<3){ + saveObj.address = "" + }else{ + def address = map.address + def i = 0 + while(i<5){ + if(address && address.toString().startsWith(map.province)){ + address = address.toString().substring(map.province.toString().length()) + } + if(address && address.toString().startsWith(map.city)){ + address = address.toString().substring(map.city.toString().length()) + } - if(address && saveObj.district){ - def tjjObj = mzbRegionMap.get(saveObj.district) - if(tjjObj){ - def alias = tjjObj.alias - if(!alias){ - alias = [] + if(address && saveObj.district){ + def tjjObj = mzbRegionMap.get(saveObj.district) + if(tjjObj){ + def alias = null + if(tjjObj.alias){ + alias = new CopyOnWriteArrayList<>(tjjObj.alias); + }else{ + alias = new CopyOnWriteArrayList<>(); + } + + if(!alias.contains(saveObj.district)){ + alias << saveObj.district + } + if(alias.size()>=2){ + alias = alias.sort(){ a, b -> + b.length().compareTo(a.length()); } - if(!alias.contains(saveObj.district)){ - alias << saveObj.district - } -// if(alias.size()>=2){ -// alias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } + } - for(def regionName : alias){ - if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ - address = address.substring(regionName.toString().length()) - break - } + for(def regionName : alias){ + if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ + address = address.substring(regionName.toString().length()) + break } } } + } - if(address && address.toString().startsWith("市辖区")){ - address = address.toString().substring(3) - } + if(address && address.toString().startsWith("市辖区")){ + address = address.toString().substring(3) + } - if(address && towns){ - for(def town : towns){ - def townAlias = town.alias - if(!townAlias){ - townAlias = [] + if(address && towns){ + for(def town : towns){ + def townAlias = null + if(town.alias){ + townAlias = new CopyOnWriteArrayList<>(town.alias); + }else{ + townAlias = new CopyOnWriteArrayList<>(); + } + + if(!townAlias.contains(town.street)){ + townAlias << town.street + } + if(townAlias.size()>=2){ + townAlias = townAlias.sort(){ a, b -> + b.length().compareTo(a.length()); + } } - if(!townAlias.contains(town.street)){ - townAlias << town.street - } -// if(townAlias.size()>=2){ -// townAlias = townAlias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } - for(def townName : townAlias){ - if(address.startsWith(townName)){ - address = address.substring(townName.toString().length()) + for(def townName : townAlias){ + if(address.startsWith(townName)){ + address = address.substring(townName.toString().length()) // if(!saveObj.township){ // saveObj.township = town.street // } - break - } + break } } } - i++ - } - if(address && address.startsWith("(") && address.endsWith(")")){ - saveObj.address = address.substring(1, address.length()-1) - }else{ - saveObj.address = address } + i++ + } + if(address && address.startsWith("(") && address.endsWith(")")){ + saveObj.address = address.substring(1, address.length()-1) + }else{ + saveObj.address = address } } return saveObj @@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def save(rows){ BulkRequest request = new BulkRequest(); - int batchSize = 1000; + int batchSize = 2000; int processedCount = 0; def client = ElasticUtils89.getClient("192.168.0.89") From f8bc248feb94baf97d78f2a7c47cffe6fa9f1d8c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 28 Nov 2024 14:03:53 +0800 Subject: [PATCH 19/20] =?UTF-8?q?=E6=96=B0poi=E4=B9=9D=E7=BA=A7=E6=8B=86?= =?UTF-8?q?=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/PoiNineSplitBatch.groovy | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy new file mode 100644 index 000000000..c5ea6d76d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy @@ -0,0 +1,54 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils + +@BatchOwner(Watcher.GaoJuan) +class PoiNineSplitBatch extends BaseBatch{ + + static main(args) { + runBatch(new PoiNineSplitBatch(), args) + } + + @Override + void process(Object args) { + run() + } + def run(){ + def i = 0; + while(i<360) { + def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\new_poi\\new_poi_address_split_${i}.xlsx" + } + + File file = new File(fileName); + if (file.exists()) { + ExcelUtils.multiThreadingModify(fileName, { d -> + if (d['C'] && !d['G']) { + println d['C'] +" "+ d['E'] + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E']) + if (jsonObject) { + if (jsonObject.get("street") && jsonObject.get("street") != "null") { + d['G'] = jsonObject.get("street") + } + if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") { + d['H'] = jsonObject.get("streetNumber") + } + if (jsonObject.get("township") && jsonObject.get("township") != "null") { + d['I'] = jsonObject.get("township") + } + } + } + }) + println fileName +" 九级拆分完成" + }else{ + println fileName +" 文件不存在" + } + i++ + } + } +} From d0cd6e575ecb640e872134ccff3f343b84fae138 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 28 Nov 2024 17:36:46 +0800 Subject: [PATCH 20/20] update --- .../script/ccbevaluation/AddSzDealData.groovy | 5 +- .../data_handle/AddDistrictShapeBatch.groovy | 18 +- .../data_handle/AutoAddDistrictBatch.groovy | 29 +-- .../script/gj/data_handle/DevProcess.groovy | 43 ++++- .../gj/data_handle/DevUpdateTestBatch.groovy | 5 +- .../script/gj/data_handle/NewPoiData.groovy | 175 ++++++++++++++++++ .../gj/data_handle/PoiNineSplitBatch.groovy | 18 +- .../fangjia/script/gj/data_handle/Test.groovy | 131 +++++-------- .../gj/zhongshu/DistrictArrountOffice.groovy | 134 ++++++++++++++ 9 files changed, 428 insertions(+), 130 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/zhongshu/DistrictArrountOffice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 0bf6ec4b9..4473e1e49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -1,15 +1,14 @@ package com.fangjia.script.ccbevaluation -import com.fangjia.script.util.DateUtils + import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection - /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy index 12879d5ed..72d64ea09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{ DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + + public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL) @@ -54,8 +57,8 @@ class AddDistrictShapeBatch extends BaseBatch{ // // } -// checkRangeCount() - synch100() + checkRangeCount() +// synch100() } def baiduMatch(city,districts){ @@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{ def checkRangeCount(){ - def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100) +// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100) + def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","乡镇街道")).batchSize(100) def count = 0 def index =0 for(def d : dbcursor){ index ++ - if(d.location){ - def coordinates = d.location.coordinates + if(d.shape){ + def coordinates = d.shape.coordinates if(coordinates.size()>1){ count++ - println count +" "+ d.city +" "+d.name + println count +" "+ d.city +" "+d.street } }else{ - println d.city + " "+d.name + println d.city + " "+d.street } if(index%1000==0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index 5367882e6..809fa216c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -35,22 +35,22 @@ class AutoAddDistrictBatch extends BaseBatch{ @Override void process(Object args) { - //查找第三方匹配没有匹配上的小区 - def citys = CommonDatautils.findAllCity() - def list = [] - list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] - for(def c : citys){ - def noMatchDatas = findNoMatchResult(c) - println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size() - def rets = checkDatas(noMatchDatas.keySet()) - list.addAll(rets) - } +// //查找第三方匹配没有匹配上的小区 +// def citys = CommonDatautils.findAllCity() +// def list = [] +// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"] +// for(def c : citys){ +// def noMatchDatas = findNoMatchResult(c) +// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size() +// def rets = checkDatas(noMatchDatas.keySet()) +// list.addAll(rets) +// } def fileName = "/home/goojia/gj/add_dist_info.xlsx" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx" + fileName = "D:\\数据\\add_dist_info_1.xlsx" } - ExcelUtils.write(fileName, list, "待入库数据") +// ExcelUtils.write(fileName, list, "待入库数据") check(fileName) @@ -61,6 +61,11 @@ class AutoAddDistrictBatch extends BaseBatch{ def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district") if(dists){ d['J'] = "库中已存在" + }else{ + dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district") + if(dists){ + d['J'] = "库中别名中存在" + } } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index f722db94c..acf416bff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils -import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.linxsh.util.AddressUtil -import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject -import org.apache.poi.ss.util.RegionUtil import org.bson.types.ObjectId -import org.joda.time.Instant -import java.time.format.DateTimeFormatter import java.util.regex.Pattern class DevProcess { @@ -64,6 +58,7 @@ class DevProcess { // task.deleteAddr() // task.deleteAddr2() task.synchDevTo100() +// task.statShape() // task.addNewDev2() // task.addAlias() @@ -87,6 +82,8 @@ class DevProcess { // task.matchDevShape()//匹配开发区围栏 // task.addQiDevResult() //获取齐梁数据结果 + +// task.updateDevShape()//修改开发区结构 } @@ -794,6 +791,18 @@ class DevProcess { development_zone_CONN.insert(rets) } + def statShape(){ + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true))) + def index = 0 + for(def r : dbcursor){ + index ++ + def coordinates = r.shape.coordinates + println index+" "+r.name+" "+coordinates.size() + + + } + } + def matchDevAndAddAddress(){ ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d-> @@ -1148,4 +1157,26 @@ class DevProcess { } },0) } + + def updateDevShape(){ + def q = new BasicDBObject() + q.put("shape", new BasicDBObject('$exists', true)) + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q) + for(def d : dbcursor){ + if(d.name!="福州长乐国际机场综合保税区"){ + def udata = new BasicDBObject() + if(d.shape.coordinates){ + d.shape.coordinates = [d.shape.coordinates] + udata.put("shape", d.shape) + } + if(d.bd_shape && d.bd_shape.coordinates){ + d.bd_shape.coordinates = [d.bd_shape.coordinates] + udata.put("bd_shape", d.bd_shape) + } + def updateQuery = new BasicDBObject("_id",d._id) + DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 49d2547b7..aaf80284e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.AMQP import net.sf.json.JSONObject import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -33,7 +32,7 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() - def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx + def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx // task.addDev(path) task.addDevAddress(path) // @@ -168,7 +167,7 @@ class DevUpdateTestBatch { println d['A'] +" 不存在" } } - },2) + },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy new file mode 100644 index 000000000..767209f83 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy @@ -0,0 +1,175 @@ +package com.fangjia.script.gj.data_handle + + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.mongodb.BasicDBObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class NewPoiData { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + def task = new NewPoiData() + task.run() + } + def run(){ +// export() +// exportNetWorkOffice() +// matchByCity() +// noMatchData() + calcuteDistance() + } + + def noMatchData(){ + ExcelUtils.multiThreadingModify("d:\\数据\\未匹配写字楼2.xlsx", { d -> + if(d['H']!='地图'){ + def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C']) + if(map){ + d['E'] = map.lng + d['F'] = map.lat + } + } + }) + } + def matchByCity(){ + ExcelUtils.multiThreadingModify("d:\\数据\\入库写字楼.xlsx",{d-> + def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false) + def formatAddress = null + if(d['D']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['D']) + d['G'] = formatAddress + } + if(!jsonObject && formatAddress){ + if(DataUtil.isPreciseAddress(formatAddress)){ + def address = d['A'] + if(d['B']){ + address = address + d['B'] + } + address = address + formatAddress + jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false) + } + } + if(jsonObject) { + d['H'] = jsonObject.get("id") + d['I'] = jsonObject.get("city") + d['J'] = jsonObject.get("district") + d['K'] = jsonObject.get("name") + d['L'] = jsonObject.get("propertyType") + d['M'] = jsonObject.get("credit") + } + }) + } + + + def export(){ + def rets = [] + rets << ["城市", "区域", "名称", "地址", "经度", "纬度"] + rets.addAll(CommonDatautils.findNewPoiDatas()) + ExcelUtils.write("d:\\数据\\poi写字楼.xlsx", rets, "写字楼") + } + + def exportNetWorkOffice(){ + def rets = [] + rets << ["城市", "区域", "名称", "地址", "经度", "纬度"] + rets.addAll(CommonDatautils.findThirdNetWorkOffice()) + ExcelUtils.write("d:\\数据\\poi写字楼-网站.xlsx", rets, "写字楼") + } + + def calcuteDistance(){ + ExcelUtils.multiThreadingModify("d:\\数据\\写字楼测试-1118-新增.xlsx", { d -> + if(d['F']){ + if(d['A'].contains("北京")){ + def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075) + d['K'] = distance + }else if(d['A'].contains("烟台")){ + def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616) + d['K'] = distance + } + } + }, 1) + } + + + + + def addDistrict(){ + def path = "D:\\数据\\入库写字楼.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def city = d['A'] + def region = d['B'] + def name = d['C'] + + def formatAddress = null + def address = d['D'] + if(address){ + if(d['G']){ + formatAddress = AddressUtil.formatAddress(city, address) + } + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + def key = city+"##"+region+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, region, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","office") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",region) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + } + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy index c5ea6d76d..21432c057 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy @@ -1,22 +1,20 @@ package com.fangjia.script.gj.data_handle -import com.fangjia.script.BaseBatch -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher + import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils -@BatchOwner(Watcher.GaoJuan) -class PoiNineSplitBatch extends BaseBatch{ +class PoiNineSplitBatch{ static main(args) { - runBatch(new PoiNineSplitBatch(), args) + def task = new PoiNineSplitBatch() + task.run() } - @Override - void process(Object args) { - run() - } +// @Override +// void process(Object args) { +// run() +// } def run(){ def i = 0; while(i<360) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy index 4d77b4cdf..3f08229ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy @@ -1,105 +1,58 @@ package com.fangjia.script.gj.data_handle -import com.fangjia.script.ccbevaluation.CcbCommonUtils -import com.fangjia.script.gj.ccb.CcbCodeUtils -import com.google.common.hash.BloomFilter -import com.google.common.hash.Funnels +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang.math.RandomUtils -import java.nio.charset.StandardCharsets -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.CopyOnWriteArrayList -import java.util.concurrent.CountDownLatch -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.atomic.AtomicLong +def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; +DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl) -class Test{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) -// 最大重试次数 - private static final int MAX_TRY_COUNT = 5; +def city = "昌江" +ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d-> + def date = d['B'] + def start = DateUtils.getFirstDayOfMonth(date) + def end = DateUtils.getLastDayOfMonth(date) + def weekends = DateUtils.getWeekendBetweenDates(start, end) + for(def weekend : weekends){ + def avg_price = d['I'] + def val = random(-200,200) + def new_price = (int)(avg_price*val/10000 + avg_price) + def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7) + def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend) - static main(args) { - def task = new Test() - task.run() - } - - def run(){ - def i=1 - while(i<2){ - def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); - def rets = process(districts) - println i + " "+districts.size() +" "+rets.size() - i++ + def o = new BasicDBObject() + o.put("avg_price", new_price) + o.put("c_date" , new Date()) + o.put("city", city) + o.put("m_date" , new Date()) + o.put("weekend", weekend) + o.put("cat","城市") + o.put("moving_average" , (int)avg_price) + if(!last_price){ + o.put("weekend_markup" , 0) + }else{ + def weekend_markup = (int)((new_price-last_price)*100/last_price) + o.put("weekend_markup" , weekend_markup) } + def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend) + REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false) } - - def process(districts){ - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(districts.size()) - - - def rets = new CopyOnWriteArrayList() - for(def i=0;i + def locations = v.split(",") + def arroundOffices = CommonDatautils.getSeaweedArroundByCat(k, locations[0], locations[1], "office") + println k +" "+arroundOffices.size() + if(arroundOffices){ + def index = 0 + for(def o : arroundOffices){ + index++ + def priceMap = findEsOffice(o.city, o.region, o.name) + if(!priceMap){ + priceMap = findEsHouse(o, "20231112") + } + def start_price = null + def end_price = null + def source = null + if(priceMap){ + if(priceMap.avg_start_price){ + start_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_start_price) + } + if(priceMap.avg_end_price){ + end_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_end_price) + } + source = priceMap.source + } + def lng = DataHandleUtils.getDoubleRound(6, o.lng/10000000000.0) + def lat = DataHandleUtils.getDoubleRound(6, o.lat/10000000000.0) + def row = [o.city, o.region, o.name, o.address, lng, lat, o.bed_kind, start_price, end_price, source] + rets << row + println index +" "+ row + } + } + } + ExcelUtils.write("d:\\数据\\写字楼测试-1118.xlsx", rets, "数据") + } + + + def findEsOffice(city, region, name){ + def map = [:] + def start_rent_prices = [] + def end_rent_prices = [] + def rent_prices = [] + + ElasticUtils.executeSqlQuery("select * from office where city='${city}' and region='${region}' and name='${name}'", { r-> + if(r.start_rent_price){ + start_rent_prices << r.start_rent_price + } + if(r.end_rent_price){ + end_rent_prices << r.end_rent_price + } + if(r.rent_price){ + rent_prices << r.rent_price + } + + }) + def avg_start_price = null + if(start_rent_prices){ + avg_start_price = start_rent_prices.sum()/start_rent_prices.size() + } + def avg_end_price = null + if(end_rent_prices){ + avg_end_price = end_rent_prices.sum()/end_rent_prices.size() + } + + if(!avg_start_price && !avg_end_price){ + if(rent_prices){ + if(rent_prices.size()>1){ + rent_prices = rent_prices.sort() + avg_start_price = rent_prices[0] + avg_end_price = rent_prices[rent_prices.size()-1] + }else{ + avg_start_price = rent_prices[0] - rent_prices[0]*0.2 + avg_end_price = rent_prices[0] + rent_prices[0]*0.2 + } + } + } + + if(avg_start_price){ + map.put("avg_start_price", avg_start_price) + } + if(avg_end_price){ + map.put("avg_end_price", avg_end_price) + } + if(map){ + map.source = "非房源" + } + return map + } + + def findEsHouse(o, startDate){ + def rent_prices = CommonDatautils.findHousePriceByDate(o.city, o.region, o.name, o.alias, "office_rent", startDate) + def avg_start_price = null + def avg_end_price = null + def map = [:] + if(rent_prices){ + if(rent_prices.size()>1){ + rent_prices = rent_prices.sort() + avg_start_price = rent_prices[0] + avg_end_price = rent_prices[rent_prices.size()-1] + }else{ + avg_start_price = rent_prices[0] - rent_prices[0]*0.2 + avg_end_price = rent_prices[0] + rent_prices[0]*0.2 + } + } + if(avg_start_price){ + map.put("avg_start_price", avg_start_price) + } + if(avg_end_price){ + map.put("avg_end_price", avg_end_price) + } + if(map){ + map.source = "房源" + } + return map + } + +}