From 74570ced7e8e6dd7eb65d0fe7521b3b39ab3e059 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 28 Aug 2021 12:31:06 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 19 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 3 +- .../ccbevaluation/BigdataPoiBatch.groovy | 66 ++--- .../ccbevaluation/DistrictDelete.groovy | 226 ++++++++++++++++++ 4 files changed, 272 insertions(+), 42 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index b56aa4f6b..4a508a01a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -27,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = null; // if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) // }else{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } @@ -407,15 +408,15 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i=startPrice && obj.price<=endPrice){ rets << obj; @@ -466,7 +467,7 @@ class BigdataDealBatch extends BaseBatch{ q.put("city", city) q.put("weekend", weekend) def dbcursor = DIST_SELL_STAT.find(q) - def map = [:] + def map = new ConcurrentHashMap() for(def d : dbcursor){ map.put(d.region+"@"+d.district_name, d.avg_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 162503f04..236ce7f12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -73,7 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); } } @@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{ if (area == null) { return false; } - if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { return true; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index f80fa70dc..8a6e106ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def citys = CcbCommonUtils.citys; // def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" -// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) -// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) + def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) + def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) -// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) -// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) + def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) + def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) // def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) - def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1) def rets = [] - def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校") +// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// rets.addAll(rows1) + + def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") rets.addAll(rows1) -// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") -// rets.addAll(rows1) -// -// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") -// rets.addAll(rows2) + def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") + rets.addAll(rows2) // // def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // rets.addAll(rows3) -// -// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") -// rets.addAll(rows4) -// -// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") -// rets.addAll(rows5) + + def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") + rets.addAll(rows4) + + def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") + rets.addAll(rows5) // // def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") // rets.addAll(rows6) - def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] def fileName = "/home/goojia/ccb/bigdata_poi.txt" if(CommonDatautils.isWindow()){ @@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - if(citys.contains(d['A'])){ + if(citys.contains(d['D'])){ def ret = addValue(d, cityCodeDataMap, cityDistMap, type) if(ret){ rets << ret @@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{ ret << "" ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << d['C'] + ret << d['F'] ret << "FJW" return ret; @@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{ def addValue(d, cityCodeDataMap, cityDistMap, type){ def ret = [] - def citycode = cityCodeDataMap.get(d['A']) + def citycode = cityCodeDataMap.get(d['D']) ret << citycode - def distCodeMap = cityDistMap.get(d['A']) - def region =RegionData.formatRegion(d['A'], d['B']) + def distCodeMap = cityDistMap.get(d['D']) + def region =RegionData.formatRegion(d['D'], d['E']) if(region){ @@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{ ret << distObj.district_code }else{ ret << "" - println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F'] + println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F'] } }else{ ret << "" } - ret << new ObjectId() - ret << d['C'] + ret << d['A'] + ret << d['F'] ret << type if(type=="地铁"){ @@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{ }else if(type =="丧葬服务"){ ret << "ob101" }else if(type =="学校"){ -// if(d['F'].contains("中学")){ + if(d['F'].contains("中学")){ ret << "ed138" -// }else if(d['F'].contains("小学")){ -// ret << "ed114" -// } + }else if(d['F'].contains("小学")){ + ret << "ed114" + } } if(d['I']){ - def lat = d['J'].toDouble() - def lng = d['I'].toDouble() + def lat = d['I'].toDouble() + def lng = d['H'].toDouble() def loactionMap = GeoUtils.gcjToGps84(lat, lng) if(loactionMap){ ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) @@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{ ret << "" ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "" + ret << "" ret << "FJW" return ret; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy new file mode 100644 index 000000000..f340f72ab --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -0,0 +1,226 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.lucene.queryparser.classic.QueryParser +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class DistrictDelete { + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new DistrictDelete() +// task.run() + task.gzRun(); + } + + def run(){ + //"泉州","深圳", "重庆", + def citys = ["广州"];//"广州", + + def delIndex = 0 + for(def city : citys){ + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"]) + + checkDist(city, dists) + } + } + + def checkDist(city,dists){ + + def rets = Collections.synchronizedList(new ArrayList()); + def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd") + + def totalCount = new AtomicLong() + def delIndex = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + println city +" 小区量:"+dists.size() + + def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"] + for(def i=0;i + count = r.count + }); + return count + } + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses.size(); + } + + + static def findHouseCount(def city, def region, def name, def alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + def aliasStr = escapeExprSpecialWord(alias.join("','")) + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and district_name in ('${aliasStr}')") + sqlBuffer.append(" and (cat = 'rent' or cat='sell')") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def count = null + while (rs.next()){ + count = rs.getInt("count") + } + return count; + } + + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def gzRun(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d-> + if(d['U'] && d['U'].toString().toDouble().toInteger()==1){ + def id = d['E'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + index++ + println index + } + },0) + } + +}