From 048e99034cfc50cf9cc53e2adc2f45ebc2420e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 28 Aug 2021 10:03:46 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SnapshotMonitor.groovy | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy new file mode 100644 index 000000000..fecb21729 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil + +/** + * 快照监控 + * + * @author shenjian + * @since 2021/8/27 + */ +@BatchOwner(Watcher.ShenJian) +class SnapshotMonitor extends BaseBatch { + + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + + static void main(String[] args) { + runBatch(new SnapshotMonitor(), args) + } + + + @Override + void process(Object args) { + // /_snapshot/es_backup/weekly-snap-2021.05* + // 监控两个月的快照 + String[] months = [DateUtils.formatDateByPattern(new Date(), "yyyy.MM"), DateUtils.formatDateByPattern(DateUtil.addMonthToDate(new Date(), -1), "yyyy.MM")] + String urlPrefix = "http://192.168.0.206:9200/_snapshot/es_backup/weekly-snap-" + + String msg = ""; + for (String month : months) { + // 必须分开写,不能有空格 + String[] cmds = ["curl", "-XGET", urlPrefix + month + "*", "--user", "elastic:goojia123456"] + String result = execCurl(cmds) + JSONObject jsonObject = JSONObject.parse(result); + + if (jsonObject.get("snapshots") != null) { + com.alibaba.fastjson.JSONArray jsonArray = jsonObject.get("snapshots") + for (JSONObject obj : jsonArray) { + String state = obj.get("state") + if (!"SUCCESS".equals(state)) { + msg = "【ES集群快照异常】\n" + obj.get("failures").toString() + DingDingMessageUtil.sendTextMessage(access_token, msg) + return ; + } + } + } + } + } + + static String execCurl(String[] cmds) { + ProcessBuilder process = new ProcessBuilder(cmds); + Process p; + try { + p = process.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); + StringBuilder builder = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + builder.append(line); + builder.append(System.getProperty("line.separator")); + } + return builder.toString(); + + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } +} From 74570ced7e8e6dd7eb65d0fe7521b3b39ab3e059 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 28 Aug 2021 12:31:06 +0800 Subject: [PATCH 2/6] update --- .../ccbevaluation/BigdataDealBatch.groovy | 19 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 3 +- .../ccbevaluation/BigdataPoiBatch.groovy | 66 ++--- .../ccbevaluation/DistrictDelete.groovy | 226 ++++++++++++++++++ 4 files changed, 272 insertions(+), 42 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index b56aa4f6b..4a508a01a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -27,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = null; // if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) // }else{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } @@ -407,15 +408,15 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i=startPrice && obj.price<=endPrice){ rets << obj; @@ -466,7 +467,7 @@ class BigdataDealBatch extends BaseBatch{ q.put("city", city) q.put("weekend", weekend) def dbcursor = DIST_SELL_STAT.find(q) - def map = [:] + def map = new ConcurrentHashMap() for(def d : dbcursor){ map.put(d.region+"@"+d.district_name, d.avg_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 162503f04..236ce7f12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -73,7 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); } } @@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{ if (area == null) { return false; } - if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { return true; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index f80fa70dc..8a6e106ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def citys = CcbCommonUtils.citys; // def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" -// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) -// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) + def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) + def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) -// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) -// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) + def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) + def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) // def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) - def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1) def rets = [] - def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校") +// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// rets.addAll(rows1) + + def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") rets.addAll(rows1) -// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") -// rets.addAll(rows1) -// -// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") -// rets.addAll(rows2) + def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") + rets.addAll(rows2) // // def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // rets.addAll(rows3) -// -// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") -// rets.addAll(rows4) -// -// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") -// rets.addAll(rows5) + + def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") + rets.addAll(rows4) + + def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") + rets.addAll(rows5) // // def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") // rets.addAll(rows6) - def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] def fileName = "/home/goojia/ccb/bigdata_poi.txt" if(CommonDatautils.isWindow()){ @@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - if(citys.contains(d['A'])){ + if(citys.contains(d['D'])){ def ret = addValue(d, cityCodeDataMap, cityDistMap, type) if(ret){ rets << ret @@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{ ret << "" ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << d['C'] + ret << d['F'] ret << "FJW" return ret; @@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{ def addValue(d, cityCodeDataMap, cityDistMap, type){ def ret = [] - def citycode = cityCodeDataMap.get(d['A']) + def citycode = cityCodeDataMap.get(d['D']) ret << citycode - def distCodeMap = cityDistMap.get(d['A']) - def region =RegionData.formatRegion(d['A'], d['B']) + def distCodeMap = cityDistMap.get(d['D']) + def region =RegionData.formatRegion(d['D'], d['E']) if(region){ @@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{ ret << distObj.district_code }else{ ret << "" - println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F'] + println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F'] } }else{ ret << "" } - ret << new ObjectId() - ret << d['C'] + ret << d['A'] + ret << d['F'] ret << type if(type=="地铁"){ @@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{ }else if(type =="丧葬服务"){ ret << "ob101" }else if(type =="学校"){ -// if(d['F'].contains("中学")){ + if(d['F'].contains("中学")){ ret << "ed138" -// }else if(d['F'].contains("小学")){ -// ret << "ed114" -// } + }else if(d['F'].contains("小学")){ + ret << "ed114" + } } if(d['I']){ - def lat = d['J'].toDouble() - def lng = d['I'].toDouble() + def lat = d['I'].toDouble() + def lng = d['H'].toDouble() def loactionMap = GeoUtils.gcjToGps84(lat, lng) if(loactionMap){ ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) @@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{ ret << "" ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "" + ret << "" ret << "FJW" return ret; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy new file mode 100644 index 000000000..f340f72ab --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -0,0 +1,226 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.lucene.queryparser.classic.QueryParser +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class DistrictDelete { + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new DistrictDelete() +// task.run() + task.gzRun(); + } + + def run(){ + //"泉州","深圳", "重庆", + def citys = ["广州"];//"广州", + + def delIndex = 0 + for(def city : citys){ + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"]) + + checkDist(city, dists) + } + } + + def checkDist(city,dists){ + + def rets = Collections.synchronizedList(new ArrayList()); + def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd") + + def totalCount = new AtomicLong() + def delIndex = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + println city +" 小区量:"+dists.size() + + def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"] + for(def i=0;i + count = r.count + }); + return count + } + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses.size(); + } + + + static def findHouseCount(def city, def region, def name, def alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + def aliasStr = escapeExprSpecialWord(alias.join("','")) + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and district_name in ('${aliasStr}')") + sqlBuffer.append(" and (cat = 'rent' or cat='sell')") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def count = null + while (rs.next()){ + count = rs.getInt("count") + } + return count; + } + + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def gzRun(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d-> + if(d['U'] && d['U'].toString().toDouble().toInteger()==1){ + def id = d['E'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + index++ + println index + } + },0) + } + +} From f530cb1a2076ba29d830729adee1f68c96503a4b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 28 Aug 2021 16:55:39 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E8=BF=87=E6=BB=A4=E8=8C=83=E5=9B=B4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 4a508a01a..0df1cfbed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -416,8 +416,13 @@ class BigdataDealBatch extends BaseBatch{ def startPrice = null def endPrice = null if(price){ - startPrice = (int)(price - price*0.1) - endPrice = (int)(price*0.5 + price) + if(dist.city=="上海"){ + startPrice = (int)(price - price*0.1) + endPrice = (int)(price*0.5 + price) + }else{ + startPrice = (int)(price - price*0.5) + endPrice = (int)(price*0.5 + price) + } } def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice) if(trades){ @@ -448,8 +453,15 @@ class BigdataDealBatch extends BaseBatch{ def price = statMap.get(obj.fjRegion+"@"+obj.fjName) if(price){ - def startPrice = (int)(price - price*0.1) - def endPrice= (int)(price*0.5 + price) + def startPrice = 0 + def endPrice= 0 + if(obj.fjCity=="上海"){ + startPrice = (int)(price - price*0.1) + endPrice = (int)(price*0.5 + price) + }else{ + startPrice = (int)(price - price*0.5) + endPrice = (int)(price*0.5 + price) + } if(obj.price>=startPrice && obj.price<=endPrice){ rets << obj; } From b140064750d57fc5dae42502b122b5446645fa12 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 2 Sep 2021 20:54:38 +0800 Subject: [PATCH 4/6] update --- .../script/ccbevaluation/BigdataCommunityBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 8 ++++---- .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 3 ++- .../com/fangjia/script/gj/district/NameProcess.groovy | 2 +- .../script/gj/district/SeaweedDataMatchProcess.groovy | 4 +--- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 97cba74f5..5b719c29e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -34,7 +34,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["宁波"]// CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 0df1cfbed..8841dbcbe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -420,8 +420,8 @@ class BigdataDealBatch extends BaseBatch{ startPrice = (int)(price - price*0.1) endPrice = (int)(price*0.5 + price) }else{ - startPrice = (int)(price - price*0.5) - endPrice = (int)(price*0.5 + price) + startPrice = 1//(int)(price - price*0.5) + endPrice = 500000//(int)(price*0.5 + price) } } def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice) @@ -459,8 +459,8 @@ class BigdataDealBatch extends BaseBatch{ startPrice = (int)(price - price*0.1) endPrice = (int)(price*0.5 + price) }else{ - startPrice = (int)(price - price*0.5) - endPrice = (int)(price*0.5 + price) + startPrice = 1//(int)(price - price*0.5) + endPrice = 500000//(int)(price*0.5 + price) } if(obj.price>=startPrice && obj.price<=endPrice){ rets << obj; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 933425df2..e80d7fd22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -174,6 +174,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); +// q.put("name", "美辰香醒") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -186,7 +187,7 @@ class CcbCommonUtils { def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def rets = [] for(def d : dbsour){ - if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){ + if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ if(d.address){ d.address = d.address.replaceAll("\r|\n|\\s", ""); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 3b2c217fc..71657eea3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -107,7 +107,7 @@ class NameProcess { } } if(name && !name.contains("路") && !name.contains("街") && !name.contains("大道") && !name.contains("巷")){ - def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(号|排)")) + def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(排)")) if(h1){ name = name.replaceAll(h1,"") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index b6d403059..38c00f546 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -24,6 +24,7 @@ class SeaweedDataMatchProcess { rMap.estate_type2 = obj.estate_type2 rMap.city = obj.city rMap.region = obj.region + rMap.alias = obj.alias rMap.name = obj.name rMap.lat = obj.lat/10000000000.0 rMap.lng = obj.lng/10000000000.0 @@ -43,9 +44,6 @@ class SeaweedDataMatchProcess { } } - - - // if(rMap && rMap.id && region.contains(rMap.region)) { // def obj = CommonDatautils.findDistrictById(rMap.id.toString()); // if (obj) { From 5e511d73964872cd41ac1cc5a8b9d983159de47c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 5 Sep 2021 11:11:50 +0800 Subject: [PATCH 5/6] dat --- .../src/main/java/com/fangjia/core/util/BaiDuApiUtils.java | 1 + .../script/ccbevaluation/BigdataCommunityBatch.groovy | 4 ++-- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index 582a71f23..4f59fe761 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -274,6 +274,7 @@ public class BaiDuApiUtils { } return json.getJSONArray("result").getJSONObject(0); }catch (Exception e){ + System.out.println("错误:"+result); e.printStackTrace(); } return null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 5b719c29e..c2efd484b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -53,9 +53,9 @@ class BigdataCommunityBatch extends BaseBatch{ // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } - def fileName = "/home/goojia/ccb/bigdata_community.txt" + def fileName = "/home/goojia/ccb/bigdata_community.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat" } def allDatas = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 236ce7f12..b8af43278 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831); } } @@ -144,9 +144,9 @@ class BigdataForsaleBatch extends BaseBatch{ } latch.await() - def fileName = "/home/goojia/ccb/bigdata_forsale.txt" + def fileName = "/home/goojia/ccb/bigdata_forsale.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.dat" } println city + " 房源总量:"+retMap.size() From 1e73566aa43934c48c56c5e16f8dcdd8170991db Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 5 Sep 2021 21:49:12 +0800 Subject: [PATCH 6/6] update --- .../com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 8841dbcbe..82766d351 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -70,9 +70,9 @@ class BigdataDealBatch extends BaseBatch{ // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") } - def fileName = "/home/goojia/ccb/bigdata_deal.txt" + def fileName = "/home/goojia/ccb/bigdata_deal.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat" } FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);