From 816798f25b3f4375f5b254a2a05fefd6631e4d65 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 15 Nov 2024 10:15:49 +0800 Subject: [PATCH] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B7=BB=E5=8A=A0=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E8=84=9A=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddCityRegionPrice.groovy | 8 +- .../data_handle/AutoAddDistrictBatch.groovy | 54 +++++--- .../gj/data_handle/DistrictRepeatBatch.groovy | 127 +++++++++++++++++- .../gj/data_handle/RegionStandardBatch.groovy | 62 +++++++-- .../gj/data_handle/TownStandardBatch.groovy | 24 ++-- 5 files changed, 230 insertions(+), 45 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index 954e0402b..a42b84aeb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil import com.fangjia.script.averager.InsertMonitorCalcCommandBatch import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.MongoClient -import com.rabbitmq.client.AMQP class AddCityRegionPrice { @@ -24,7 +22,7 @@ class AddCityRegionPrice { static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() - task.addCityprice() +// task.addCityprice() // task.addDistrictprice() // task.addCityRentPrice() // task.addRegionRentPrice() @@ -32,7 +30,7 @@ class AddCityRegionPrice { // task.updateRegionRentData() // task.updateDealPriceAreaType() -// task.checkCityImport() + task.checkCityImport() } def addRegionPrice(){ @@ -255,7 +253,7 @@ class AddCityRegionPrice { def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1) def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1) if(startObj && endObj){ - def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409") + def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410") months.each {m-> def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger()) if(!obj){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index 8e1a842fe..80a9e2bdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - /** * 小区自动入库(新增小区,每月运行一次) */ @@ -37,7 +36,7 @@ class AutoAddDistrictBatch extends BaseBatch{ @Override void process(Object args) { //查找第三方匹配没有匹配上的小区 - def citys = ["上海"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() def list = [] list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] for(def c : citys){ @@ -52,6 +51,18 @@ class AutoAddDistrictBatch extends BaseBatch{ fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx" } ExcelUtils.write(fileName, list, "待入库数据") + + + check(fileName) + } + + def check(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district") + if(dists){ + d['J'] = "库中已存在" + } + }) } def checkDatas(keysets){ @@ -62,8 +73,8 @@ class AutoAddDistrictBatch extends BaseBatch{ def totalCount = new AtomicLong() for(def i=0;i - if(d.name == name && d.type.contains("住宅") && d.shape){ + if(d.name == name && d.type.contains("住宅") ){ //&& d.shape obj = d } }) @@ -154,6 +177,7 @@ class AutoAddDistrictBatch extends BaseBatch{ q.put("match_id", new BasicDBObject('$exists',false)) q.put("old_match_id", new BasicDBObject('$exists',false)) q.put("city", c) + q.put("name", "钱江大厦") def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) def rMap = [:] for(def d : dbcursor){ @@ -228,7 +252,7 @@ class AutoAddDistrictBatch extends BaseBatch{ q.put("district", district) q.put("name", name) q.put('std_tag', '房地产;住宅区') - def count = BAIDU_POI_COLL.count(q) - return count; + def obj = BAIDU_POI_COLL.findOne(q) + return obj; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy index 3fb5a0d48..74d0ddbcb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy @@ -1,17 +1,28 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang.SerializationUtils +import org.bson.types.ObjectId /** * 查找重复小区 */ class DistrictRepeatBatch { + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + static main(args) { def task = new DistrictRepeatBatch() - task.run() +// task.run() + task.resultProcess() } def run(){ @@ -65,4 +76,118 @@ class DistrictRepeatBatch { return rows; } + + def resultProcess(){ + ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d-> + if(d["O"] && !d["O"].contains("不处理") && !d['R']){ + if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){ + def updteId = d['H'] + def q = new BasicDBObject() + q.put("_id", new ObjectId(updteId)) + + def udata = new BasicDBObject() + if(d['O']=="K列改为商住" ){ + udata.put("estate_type2", "商住") + }else{ + udata.put("estate_type2", "别墅") + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + saveOptLog(d["I"], d["J"], d["K"], "update") + d["R"] = "库中已处理" + } + else if(d["O"] == "都删"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata)) + saveOptLog(d["B"], d["C"], d["D"], "remove") + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata)) + saveOptLog(d["I"], d["J"], d["K"], "remove") + d["R"] = "库中已处理" + } + else if(d["O"]=="删除K列小区"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata)) + saveOptLog(d["I"], d["J"], d["K"], "remove") + + def obj = CommonDatautils.findDistrictById(d['A']) + if(d['C']==d['J']){ + def alias = [] + def addr_alias = [] + if(d['D']!=d['K']){ + def s1 = d['D'].replace(d['K'], "") + def s2 = d['K'].replace(d['D'], "") + def addNameAlias = true + if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){ + addNameAlias = false + } + if(addNameAlias){ + + if(obj.alias){ + alias = SerializationUtils.clone(obj.alias) + } + if(!alias.contains(d['K'])){ + alias << d['K'] + } + } + } + + def address = null + if(d['M'] && d['F']!=d['M']){ + def formatAddress = AddressUtil.formatAddress(d['A'], d['M']) + if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){ + if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){ + if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){ + address = formatAddress + }else{ + if(obj.addr_alias){ + addr_alias = SerializationUtils.clone(obj.addr_alias) + } + if(!addr_alias.contains(formatAddress)){ + addr_alias << formatAddress + } + } + } + } + } + + if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){ + def udata2 = new BasicDBObject() + if(alias){ + udata2.put("alias", alias) + } + if(addr_alias){ + udata2.put("addr_alias", addr_alias) + } + if(address){ + udata2.put("address", address) + } + udata2.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2)) + } + } + + d["R"] = "库中已处理" + } + } + }) + } + + def saveOptLog(city, region, name, type){ + def obj = new BasicDBObject(); + obj.put("city",city); + obj.put("region",region); + obj.put("district_name",name); + obj.put("category","district"); + obj.put("c_date", new Date()) + obj.put("m_date", new Date()) + obj.put("type", type) + obj.put("user_name", "高娟") + obj.put("origin", "script") + OPERATION_LOG_COLL.save(obj) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index c78e0a84b..0afba947a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject +import org.apache.commons.io.FileUtils import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions @@ -41,12 +42,11 @@ class RegionStandardBatch extends BaseBatch{ def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] def index = 0 - def citys = ["攀枝花"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() for(def city : citys){ index ++ -// if(index<32){ -// continue -// } + + if(runCitys.contains(city)){ continue } @@ -57,19 +57,32 @@ class RegionStandardBatch extends BaseBatch{ } if(fullCityName){ - def rMap = regionFullNameMap.get(fullCityName) - if(rMap){ - def set = rMap.keySet() - if(set.size()>1){ - def rets = findEsPoiByCity(fullCityName) - println city +" "+rets.size() - poiHaveRegionProcess(city, set, rMap, rets) - }else{ + def fileName = "/home/goojia/gj/poi/${fullCityName}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\poi\\${fullCityName}.csv" + } + File file=new File(fileName); + if(file.exists()){ + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + if(set.size()>1){ +// def rets = findEsPoiByCity(fullCityName) + def rets = readCSV(fileName) + println city +" "+rets.size() + poiHaveRegionProcess(city, set, rMap, rets) + }else{ // haveStreetProcess(city, set, cat) + + println "没找区域:"+city + } + }else{ + println "没找区域:"+city } }else{ - println "没找区域:"+city + println fileName+" 文件没找到" } + }else{ println "没找到相应的全称:"+city } @@ -365,6 +378,29 @@ class RegionStandardBatch extends BaseBatch{ // // // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 // } + def readCSV(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + for(line in lines) { + if(line){ + String[] arrs = line.split("\\|@\\|") + def o = new BasicDBObject() + try{ + o.put("city", arrs[0]) + o.put("district", arrs[1]) + o.put("name", arrs[2]) + o.put("township", arrs[3]) + o.put("status", arrs[4]) + o.put("id", arrs[5]) + rets << o + }catch(Exception e){ + println filename+" 文件异常" + } + } + + } + return rets; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index cbe40d902..545a0d186 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -66,7 +66,7 @@ class TownStandardBatch extends BaseBatch{ def townStandardByPoi(){ updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"] - def citys = CommonDatautils.findAllCity() + def citys = ["福州"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -88,7 +88,7 @@ class TownStandardBatch extends BaseBatch{ def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"]) def set = rMap.keySet() // def rets = findEsPoiByCity(fullCityName) - def rets = readPoiCsv(fullCityName) + def rets = readPoiCsv(fileName) println city +" "+rets.size() if(set.size()>1){ poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap) @@ -96,6 +96,8 @@ class TownStandardBatch extends BaseBatch{ // haveStreetProcess(city, set, cat) } } + }else{ + println fileName+" 文件没找到" } }else{ println "没找到相应的全称:"+city @@ -201,16 +203,16 @@ class TownStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["township":updateTown]) results << m -// if(results.size()%100==0){ -// updateByEs(results) -// results.clear() -// } + if(results.size()%100==0){ + updateByEs(results) + results.clear() + } updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown] println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown }else{ def m = [:] m.put(d.id, ["township":""]) - results << m +// results << m // if(results.size()%100==0){ // updateByEs(results) // results.clear() @@ -224,10 +226,10 @@ class TownStandardBatch extends BaseBatch{ } } -// if(results.size()>0){ -// updateByEs(results) -// results.clear() -// } + if(results.size()>0){ + updateByEs(results) + results.clear() + } println fullCityName +" 处理完成" }