update
This commit is contained in:
+10
-9
@@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -27,7 +28,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
@@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = null;
|
||||
// if(citys.contains(city)){
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
|
||||
// }else{
|
||||
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }
|
||||
@@ -407,15 +408,15 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def price = statMap.get(dist.region+"@"+dist.name)
|
||||
def startPrice = null
|
||||
def endPrice = null
|
||||
if(price){
|
||||
startPrice = (int)(price - price*0.5)
|
||||
startPrice = (int)(price - price*0.1)
|
||||
endPrice = (int)(price*0.5 + price)
|
||||
}
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
@@ -428,8 +429,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// };
|
||||
// })
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
@@ -447,7 +448,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
|
||||
if(price){
|
||||
def startPrice = (int)(price - price*0.5)
|
||||
def startPrice = (int)(price - price*0.1)
|
||||
def endPrice= (int)(price*0.5 + price)
|
||||
if(obj.price>=startPrice && obj.price<=endPrice){
|
||||
rets << obj;
|
||||
@@ -466,7 +467,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = [:]
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
}
|
||||
|
||||
+1
-2
@@ -73,7 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
index++
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
|
||||
}
|
||||
}
|
||||
@@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
+35
-31
@@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
||||
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
|
||||
def rets = []
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows1)
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
rets.addAll(rows1)
|
||||
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
// rets.addAll(rows1)
|
||||
//
|
||||
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
// rets.addAll(rows2)
|
||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
rets.addAll(rows2)
|
||||
//
|
||||
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows3)
|
||||
//
|
||||
// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
// rets.addAll(rows4)
|
||||
//
|
||||
// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
// rets.addAll(rows5)
|
||||
|
||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
rets.addAll(rows4)
|
||||
|
||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
rets.addAll(rows5)
|
||||
//
|
||||
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
// rets.addAll(rows6)
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
@@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(citys.contains(d['A'])){
|
||||
if(citys.contains(d['D'])){
|
||||
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
|
||||
if(ret){
|
||||
rets << ret
|
||||
@@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << d['C']
|
||||
ret << d['F']
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
@@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
|
||||
def addValue(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
def citycode = cityCodeDataMap.get(d['D'])
|
||||
ret << citycode
|
||||
def distCodeMap = cityDistMap.get(d['A'])
|
||||
def region =RegionData.formatRegion(d['A'], d['B'])
|
||||
def distCodeMap = cityDistMap.get(d['D'])
|
||||
def region =RegionData.formatRegion(d['D'], d['E'])
|
||||
|
||||
if(region){
|
||||
|
||||
@@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
|
||||
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << new ObjectId()
|
||||
ret << d['C']
|
||||
ret << d['A']
|
||||
ret << d['F']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
@@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
// if(d['F'].contains("中学")){
|
||||
if(d['F'].contains("中学")){
|
||||
ret << "ed138"
|
||||
// }else if(d['F'].contains("小学")){
|
||||
// ret << "ed114"
|
||||
// }
|
||||
}else if(d['F'].contains("小学")){
|
||||
ret << "ed114"
|
||||
}
|
||||
}
|
||||
if(d['I']){
|
||||
def lat = d['J'].toDouble()
|
||||
def lng = d['I'].toDouble()
|
||||
def lat = d['I'].toDouble()
|
||||
def lng = d['H'].toDouble()
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
@@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class DistrictDelete {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
task.gzRun();
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"泉州","深圳", "重庆",
|
||||
def citys = ["广州"];//"广州",
|
||||
|
||||
def delIndex = 0
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
|
||||
checkDist(city, dists)
|
||||
}
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd")
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def delIndex = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
println city +" 小区量:"+dists.size()
|
||||
|
||||
def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def dist = dists[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouses(city, dist.region, "sell", 20190101,20211231, dist.name, dist.alias)
|
||||
def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def tradeCount = 0
|
||||
if(!count){
|
||||
tradeCount = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, 0, 300000)
|
||||
if(!tradeCount){
|
||||
def nameFlag = false
|
||||
for(def key : keys){
|
||||
if(dist.name.contains(key)){
|
||||
nameFlag = true
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(!nameFlag){
|
||||
if(dist.name.size()<=3 && (dist.name.contains("村") || dist.name.contains("楼") || dist.name.contains("居") )){
|
||||
nameFlag = true
|
||||
}
|
||||
}
|
||||
def fieldFlag = false
|
||||
if(dist.finish_building_date && dist.build_type){
|
||||
fieldFlag = true
|
||||
}
|
||||
|
||||
if(!tradeCount && !count && !fieldFlag){
|
||||
def rowCount = findCount(dist._id.toString())
|
||||
if(rowCount || nameFlag){
|
||||
delIndex.addAndGet(1)
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',q))
|
||||
println "[ ${delIndex.get()}/${totalCount.get()} ] "+city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
|
||||
}else{
|
||||
println city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
|
||||
}
|
||||
|
||||
rets << [dist._id.toString(),dist.city, dist.region, dist.name, dist.address]
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\多余小区_${city}.xlsx",rets, "无房源")
|
||||
}
|
||||
|
||||
|
||||
def findCount(distId){
|
||||
def sql = "select count(*) as count from fjw_community_artifi_check where commid='${distId}'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
static def findHouseCount(def city, def region, def name, def alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and region = '"+region+"'")
|
||||
sqlBuffer.append(" and district_name in ('${aliasStr}')")
|
||||
sqlBuffer.append(" and (cat = 'rent' or cat='sell')")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def count = null
|
||||
while (rs.next()){
|
||||
count = rs.getInt("count")
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def gzRun(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
|
||||
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
|
||||
def id = d['E']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
index++
|
||||
println index
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user