Merge branch 'script_gj' into 'master'
自动添加小区脚本修改 See merge request !1008
This commit is contained in:
+3
-5
@@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
|
||||
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.MongoClient
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
class AddCityRegionPrice {
|
||||
|
||||
@@ -24,7 +22,7 @@ class AddCityRegionPrice {
|
||||
static main(args) {
|
||||
def task = new AddCityRegionPrice()
|
||||
// task.addRegionPrice()
|
||||
task.addCityprice()
|
||||
// task.addCityprice()
|
||||
// task.addDistrictprice()
|
||||
// task.addCityRentPrice()
|
||||
// task.addRegionRentPrice()
|
||||
@@ -32,7 +30,7 @@ class AddCityRegionPrice {
|
||||
|
||||
// task.updateRegionRentData()
|
||||
// task.updateDealPriceAreaType()
|
||||
// task.checkCityImport()
|
||||
task.checkCityImport()
|
||||
}
|
||||
|
||||
def addRegionPrice(){
|
||||
@@ -255,7 +253,7 @@ class AddCityRegionPrice {
|
||||
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
|
||||
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
|
||||
if(startObj && endObj){
|
||||
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
|
||||
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410")
|
||||
months.each {m->
|
||||
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
|
||||
if(!obj){
|
||||
|
||||
+39
-15
@@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 小区自动入库(新增小区,每月运行一次)
|
||||
*/
|
||||
@@ -37,7 +36,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = ["上海"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||
for(def c : citys){
|
||||
@@ -52,6 +51,18 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
|
||||
|
||||
check(fileName)
|
||||
}
|
||||
|
||||
def check(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
|
||||
if(dists){
|
||||
d['J'] = "库中已存在"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkDatas(keysets){
|
||||
@@ -62,8 +73,8 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<keysets.size();i++){
|
||||
final def key = keysets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def arrs = key.split("##")
|
||||
@@ -72,18 +83,30 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
//检测有没有房源
|
||||
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
|
||||
//检查百度地图有没有
|
||||
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||
def baidu_obj = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||
def baidu_count = 0
|
||||
def baidu_type = null
|
||||
if(baidu_obj){
|
||||
baidu_count = 1
|
||||
baidu_type = baidu_obj.std_tag
|
||||
}
|
||||
//检查高德地图有没有
|
||||
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
def gaode_count = 0
|
||||
def gaode_type = null
|
||||
if(gaode_obj){
|
||||
gaode_count = 1
|
||||
gaode_type = gaode_obj.type
|
||||
}
|
||||
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
@@ -123,18 +146,18 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
if(fullDistName){
|
||||
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
|
||||
if(obj){
|
||||
return 1
|
||||
return obj
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
return null
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name && d.type.contains("住宅") && d.shape){
|
||||
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
@@ -154,6 +177,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
q.put("match_id", new BasicDBObject('$exists',false))
|
||||
q.put("old_match_id", new BasicDBObject('$exists',false))
|
||||
q.put("city", c)
|
||||
q.put("name", "钱江大厦")
|
||||
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
@@ -228,7 +252,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
def count = BAIDU_POI_COLL.count(q)
|
||||
return count;
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
+126
-1
@@ -1,17 +1,28 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.SerializationUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 查找重复小区
|
||||
*/
|
||||
class DistrictRepeatBatch {
|
||||
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictRepeatBatch()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.resultProcess()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -65,4 +76,118 @@ class DistrictRepeatBatch {
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
def resultProcess(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d->
|
||||
if(d["O"] && !d["O"].contains("不处理") && !d['R']){
|
||||
if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){
|
||||
def updteId = d['H']
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(updteId))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
if(d['O']=="K列改为商住" ){
|
||||
udata.put("estate_type2", "商住")
|
||||
}else{
|
||||
udata.put("estate_type2", "别墅")
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["I"], d["J"], d["K"], "update")
|
||||
d["R"] = "库中已处理"
|
||||
}
|
||||
else if(d["O"] == "都删"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["B"], d["C"], d["D"], "remove")
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["I"], d["J"], d["K"], "remove")
|
||||
d["R"] = "库中已处理"
|
||||
}
|
||||
else if(d["O"]=="删除K列小区"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["I"], d["J"], d["K"], "remove")
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
if(d['C']==d['J']){
|
||||
def alias = []
|
||||
def addr_alias = []
|
||||
if(d['D']!=d['K']){
|
||||
def s1 = d['D'].replace(d['K'], "")
|
||||
def s2 = d['K'].replace(d['D'], "")
|
||||
def addNameAlias = true
|
||||
if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){
|
||||
addNameAlias = false
|
||||
}
|
||||
if(addNameAlias){
|
||||
|
||||
if(obj.alias){
|
||||
alias = SerializationUtils.clone(obj.alias)
|
||||
}
|
||||
if(!alias.contains(d['K'])){
|
||||
alias << d['K']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def address = null
|
||||
if(d['M'] && d['F']!=d['M']){
|
||||
def formatAddress = AddressUtil.formatAddress(d['A'], d['M'])
|
||||
if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){
|
||||
if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){
|
||||
if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){
|
||||
address = formatAddress
|
||||
}else{
|
||||
if(obj.addr_alias){
|
||||
addr_alias = SerializationUtils.clone(obj.addr_alias)
|
||||
}
|
||||
if(!addr_alias.contains(formatAddress)){
|
||||
addr_alias << formatAddress
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){
|
||||
def udata2 = new BasicDBObject()
|
||||
if(alias){
|
||||
udata2.put("alias", alias)
|
||||
}
|
||||
if(addr_alias){
|
||||
udata2.put("addr_alias", addr_alias)
|
||||
}
|
||||
if(address){
|
||||
udata2.put("address", address)
|
||||
}
|
||||
udata2.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2))
|
||||
}
|
||||
}
|
||||
|
||||
d["R"] = "库中已处理"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, type){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","district");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", type)
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+49
-13
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
@@ -41,12 +42,11 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
|
||||
def index = 0
|
||||
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
index ++
|
||||
// if(index<32){
|
||||
// continue
|
||||
// }
|
||||
|
||||
|
||||
if(runCitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
@@ -57,19 +57,32 @@ class RegionStandardBatch extends BaseBatch{
|
||||
}
|
||||
if(fullCityName){
|
||||
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
// def rets = findEsPoiByCity(fullCityName)
|
||||
def rets = readCSV(fileName)
|
||||
println city +" "+rets.size()
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
|
||||
println "没找区域:"+city
|
||||
}
|
||||
}else{
|
||||
println "没找区域:"+city
|
||||
}
|
||||
}else{
|
||||
println "没找区域:"+city
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
@@ -365,6 +378,29 @@ class RegionStandardBatch extends BaseBatch{
|
||||
//
|
||||
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||
// }
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println filename+" 文件异常"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+13
-11
@@ -66,7 +66,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def townStandardByPoi(){
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
@@ -88,7 +88,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
|
||||
def set = rMap.keySet()
|
||||
// def rets = findEsPoiByCity(fullCityName)
|
||||
def rets = readPoiCsv(fullCityName)
|
||||
def rets = readPoiCsv(fileName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||
@@ -96,6 +96,8 @@ class TownStandardBatch extends BaseBatch{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
@@ -201,16 +203,16 @@ class TownStandardBatch extends BaseBatch{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":updateTown])
|
||||
results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
results << m
|
||||
// results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
@@ -224,10 +226,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
// if(results.size()>0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user