Merge branch 'script_gj' into 'master'

自动添加小区脚本修改

See merge request !1008
This commit is contained in:
2024-11-15 10:09:22 +08:00
5 changed files with 230 additions and 45 deletions
@@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
import com.rabbitmq.client.AMQP
class AddCityRegionPrice {
@@ -24,7 +22,7 @@ class AddCityRegionPrice {
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
task.addCityprice()
// task.addCityprice()
// task.addDistrictprice()
// task.addCityRentPrice()
// task.addRegionRentPrice()
@@ -32,7 +30,7 @@ class AddCityRegionPrice {
// task.updateRegionRentData()
// task.updateDealPriceAreaType()
// task.checkCityImport()
task.checkCityImport()
}
def addRegionPrice(){
@@ -255,7 +253,7 @@ class AddCityRegionPrice {
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410")
months.each {m->
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
if(!obj){
@@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 小区自动入库(新增小区,每月运行一次)
*/
@@ -37,7 +36,7 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = ["上海"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
@@ -52,6 +51,18 @@ class AutoAddDistrictBatch extends BaseBatch{
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
}
def check(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中已存在"
}
})
}
def checkDatas(keysets){
@@ -62,8 +73,8 @@ class AutoAddDistrictBatch extends BaseBatch{
def totalCount = new AtomicLong()
for(def i=0;i<keysets.size();i++){
final def key = keysets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def arrs = key.split("##")
@@ -72,18 +83,30 @@ class AutoAddDistrictBatch extends BaseBatch{
//检测有没有房源
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//检查百度地图有没有
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
def baidu_obj = findBaiduCount(arrs[0], arrs[1], arrs[2])
def baidu_count = 0
def baidu_type = null
if(baidu_obj){
baidu_count = 1
baidu_type = baidu_obj.std_tag
}
//检查高德地图有没有
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_count = 0
def gaode_type = null
if(gaode_obj){
gaode_count = 1
gaode_type = gaode_obj.type
}
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
return rets
@@ -123,18 +146,18 @@ class AutoAddDistrictBatch extends BaseBatch{
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
return 1
return obj
}
}
}
return 0
return null
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") && d.shape){
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
obj = d
}
})
@@ -154,6 +177,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("match_id", new BasicDBObject('$exists',false))
q.put("old_match_id", new BasicDBObject('$exists',false))
q.put("city", c)
q.put("name", "钱江大厦")
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
def rMap = [:]
for(def d : dbcursor){
@@ -228,7 +252,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def count = BAIDU_POI_COLL.count(q)
return count;
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
}
@@ -1,17 +1,28 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.SerializationUtils
import org.bson.types.ObjectId
/**
* 查找重复小区
*/
class DistrictRepeatBatch {
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictRepeatBatch()
task.run()
// task.run()
task.resultProcess()
}
def run(){
@@ -65,4 +76,118 @@ class DistrictRepeatBatch {
return rows;
}
def resultProcess(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d->
if(d["O"] && !d["O"].contains("不处理") && !d['R']){
if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){
def updteId = d['H']
def q = new BasicDBObject()
q.put("_id", new ObjectId(updteId))
def udata = new BasicDBObject()
if(d['O']=="K列改为商住" ){
udata.put("estate_type2", "商住")
}else{
udata.put("estate_type2", "别墅")
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "update")
d["R"] = "库中已处理"
}
else if(d["O"] == "都删"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata))
saveOptLog(d["B"], d["C"], d["D"], "remove")
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "remove")
d["R"] = "库中已处理"
}
else if(d["O"]=="删除K列小区"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "remove")
def obj = CommonDatautils.findDistrictById(d['A'])
if(d['C']==d['J']){
def alias = []
def addr_alias = []
if(d['D']!=d['K']){
def s1 = d['D'].replace(d['K'], "")
def s2 = d['K'].replace(d['D'], "")
def addNameAlias = true
if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){
addNameAlias = false
}
if(addNameAlias){
if(obj.alias){
alias = SerializationUtils.clone(obj.alias)
}
if(!alias.contains(d['K'])){
alias << d['K']
}
}
}
def address = null
if(d['M'] && d['F']!=d['M']){
def formatAddress = AddressUtil.formatAddress(d['A'], d['M'])
if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){
if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){
if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){
address = formatAddress
}else{
if(obj.addr_alias){
addr_alias = SerializationUtils.clone(obj.addr_alias)
}
if(!addr_alias.contains(formatAddress)){
addr_alias << formatAddress
}
}
}
}
}
if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){
def udata2 = new BasicDBObject()
if(alias){
udata2.put("alias", alias)
}
if(addr_alias){
udata2.put("addr_alias", addr_alias)
}
if(address){
udata2.put("address", address)
}
udata2.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2))
}
}
d["R"] = "库中已处理"
}
}
})
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", type)
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -41,12 +42,11 @@ class RegionStandardBatch extends BaseBatch{
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
def index = 0
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
for(def city : citys){
index ++
// if(index<32){
// continue
// }
if(runCitys.contains(city)){
continue
}
@@ -57,19 +57,32 @@ class RegionStandardBatch extends BaseBatch{
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
}
File file=new File(fileName);
if(file.exists()){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
// def rets = findEsPoiByCity(fullCityName)
def rets = readCSV(fileName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
println "没找区域:"+city
}
}else{
println "没找区域:"+city
}
}else{
println "没找区域:"+city
println fileName+" 文件没找到"
}
}else{
println "没找到相应的全称:"+city
}
@@ -365,6 +378,29 @@ class RegionStandardBatch extends BaseBatch{
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
try{
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}catch(Exception e){
println filename+" 文件异常"
}
}
}
return rets;
}
}
@@ -66,7 +66,7 @@ class TownStandardBatch extends BaseBatch{
def townStandardByPoi(){
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
def citys = CommonDatautils.findAllCity()
def citys = ["福州"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -88,7 +88,7 @@ class TownStandardBatch extends BaseBatch{
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
def set = rMap.keySet()
// def rets = findEsPoiByCity(fullCityName)
def rets = readPoiCsv(fullCityName)
def rets = readPoiCsv(fileName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
@@ -96,6 +96,8 @@ class TownStandardBatch extends BaseBatch{
// haveStreetProcess(city, set, cat)
}
}
}else{
println fileName+" 文件没找到"
}
}else{
println "没找到相应的全称:"+city
@@ -201,16 +203,16 @@ class TownStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
results << m
// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
@@ -224,10 +226,10 @@ class TownStandardBatch extends BaseBatch{
}
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
if(results.size()>0){
updateByEs(results)
results.clear()
}
println fullCityName +" 处理完成"
}