Merge branch 'script_gj' into 'master'

Script gj

See merge request !96
This commit is contained in:
2021-08-28 11:40:05 +08:00
4 changed files with 272 additions and 42 deletions
@@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -27,7 +28,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{
def startDate = null;
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
@@ -407,15 +408,15 @@ class BigdataDealBatch extends BaseBatch{
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
if(price){
startPrice = (int)(price - price*0.5)
startPrice = (int)(price - price*0.1)
endPrice = (int)(price*0.5 + price)
}
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
@@ -428,8 +429,8 @@ class BigdataDealBatch extends BaseBatch{
} finally {
latch.countDown()
}
// };
// })
};
})
}
latch.await()
return tradesList
@@ -447,7 +448,7 @@ class BigdataDealBatch extends BaseBatch{
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
if(price){
def startPrice = (int)(price - price*0.5)
def startPrice = (int)(price - price*0.1)
def endPrice= (int)(price*0.5 + price)
if(obj.price>=startPrice && obj.price<=endPrice){
rets << obj;
@@ -466,7 +467,7 @@ class BigdataDealBatch extends BaseBatch{
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = [:]
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
@@ -73,7 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
}
}
@@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
@@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows1)
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
// rets.addAll(rows1)
//
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
// rets.addAll(rows2)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows3)
//
// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
// rets.addAll(rows4)
//
// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
// rets.addAll(rows5)
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
if(CommonDatautils.isWindow()){
@@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['A'])){
if(citys.contains(d['D'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
@@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << d['C']
ret << d['F']
ret << "FJW"
return ret;
@@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['A'])
def citycode = cityCodeDataMap.get(d['D'])
ret << citycode
def distCodeMap = cityDistMap.get(d['A'])
def region =RegionData.formatRegion(d['A'], d['B'])
def distCodeMap = cityDistMap.get(d['D'])
def region =RegionData.formatRegion(d['D'], d['E'])
if(region){
@@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{
ret << distObj.district_code
}else{
ret << ""
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << new ObjectId()
ret << d['C']
ret << d['A']
ret << d['F']
ret << type
if(type=="地铁"){
@@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
// if(d['F'].contains("中学")){
if(d['F'].contains("中学")){
ret << "ed138"
// }else if(d['F'].contains("小学")){
// ret << "ed114"
// }
}else if(d['F'].contains("小学")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['J'].toDouble()
def lng = d['I'].toDouble()
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
@@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << "FJW"
return ret;
@@ -0,0 +1,226 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class DistrictDelete {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new DistrictDelete()
// task.run()
task.gzRun();
}
def run(){
//"泉州","深圳", "重庆",
def citys = ["广州"];//"广州",
def delIndex = 0
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
checkDist(city, dists)
}
}
def checkDist(city,dists){
def rets = Collections.synchronizedList(new ArrayList());
def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd")
def totalCount = new AtomicLong()
def delIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
println city +" 小区量:"+dists.size()
def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"]
for(def i=0;i<dists.size();i++) {
final def dist = dists[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def count = findHouses(city, dist.region, "sell", 20190101,20211231, dist.name, dist.alias)
def count = findHouseCount(city, dist.region, dist.name, dist.alias)
def tradeCount = 0
if(!count){
tradeCount = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, 0, 300000)
if(!tradeCount){
def nameFlag = false
for(def key : keys){
if(dist.name.contains(key)){
nameFlag = true
break;
}
}
if(!nameFlag){
if(dist.name.size()<=3 && (dist.name.contains("村") || dist.name.contains("楼") || dist.name.contains("居") )){
nameFlag = true
}
}
def fieldFlag = false
if(dist.finish_building_date && dist.build_type){
fieldFlag = true
}
if(!tradeCount && !count && !fieldFlag){
def rowCount = findCount(dist._id.toString())
if(rowCount || nameFlag){
delIndex.addAndGet(1)
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',q))
println "[ ${delIndex.get()}/${totalCount.get()} ] "+city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
}else{
println city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
}
rets << [dist._id.toString(),dist.city, dist.region, dist.name, dist.address]
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\建行\\房产估值\\多余小区_${city}.xlsx",rets, "无房源")
}
def findCount(distId){
def sql = "select count(*) as count from fjw_community_artifi_check where commid='${distId}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size();
}
static def findHouseCount(def city, def region, def name, def alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
def aliasStr = escapeExprSpecialWord(alias.join("','"))
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name in ('${aliasStr}')")
sqlBuffer.append(" and (cat = 'rent' or cat='sell')")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def count = null
while (rs.next()){
count = rs.getInt("count")
}
return count;
}
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def gzRun(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
def id = d['E']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
index++
println index
}
},0)
}
}