From ec54ae056d692eee25f2aaa49592824fa59640ec Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Feb 2022 17:02:25 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=BC=E6=95=B0=E6=8D=AE=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/core/util/RandomUtil.java | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 6 +- .../ccbevaluation/DistrictDelete.groovy | 180 +++++++++++++++++- .../DistrictGuidePriceBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 35 ++-- .../script/ccbevaluation/LoadGuide.groovy | 13 +- .../fangjia/script/ccbevaluation/Test.groovy | 57 +++--- .../gj/paimai/LianjiaCrawlByChrome.groovy | 26 ++- 8 files changed, 263 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java index 25f4d5a12..234c29d79 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java @@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils { public static void main(String[] args) { for (int i = 0; i < 1000; i++) { - System.out.println(random(0, 10)); + System.out.println(random(-9, 9)); } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index e38ff62f4..9cb9003d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; - for(def city : citys){ - addDealPriceMatch(city); - } +// for(def city : citys){ +// addDealPriceMatch(city); +// } println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy index 1dbc5cd4d..eaef4e568 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -1,15 +1,20 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import com.mongodb.DBObject import groovy.sql.Sql import org.apache.lucene.queryparser.classic.QueryParser +import org.apache.poi.ss.util.RegionUtil import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder @@ -37,15 +42,74 @@ class DistrictDelete { dataSource.setProperties(connectionProperties) } - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static main(args) { def task = new DistrictDelete() // task.run() // task.gzRun(); // task.deleteDistrict() - task.deleteCommPrice() +// task.deleteCommPrice() +// task.distHouseCountStat() + task.addThirdPartyStat() +// task.processRegion() + } + def processRegion(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d-> + if(d['B']){ + def region = RegionData.formatRegion("重庆",d['B']) + if(region){ + d['B'] = region + } + } + },0) + } + + def addThirdPartyStat(){ + def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1) + def idMaps = [:] + for(def d : datas){ + def id = d['A'] + idMaps.put(id, d['D']) + } + def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"]) + def totalCount = new AtomicLong() + def map = findThirdPartyPrice("上海") + final CountDownLatch latch = new CountDownLatch(dists.size()) + def thirdLists = Collections.synchronizedList(new ArrayList()); + thirdLists << ["ID","区域","名称","总量"] + for(def i=0;i3000){ + prices << db.zhugezhaofang_esf_price + } + if(db.anjuke_esf_price && db.anjuke_esf_price>3000){ + prices << db.anjuke_esf_price + } + if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + prices << db.get("58tc_esf_price") + } + if(db.soufang_esf_price && db.soufang_esf_price>3000){ + prices << db.soufang_esf_price + } + if(db.lianjia_esf_price && db.lianjia_esf_price>3000){ + prices << db.lianjia_esf_price + } + if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + prices << db.cityhouse_esf_price + } + + if(db.maitian_esf_price && db.maitian_esf_price>3000){ + prices << db.maitian_esf_price + } + + if(db.beike_esf_price && db.beike_esf_price>3000){ + prices << db.beike_esf_price + } + + if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){ + prices << db.get("365house_esf_price") + } + + if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + prices << db.zhongyuan_esf_price + } + + if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + prices << db.tuboshi_esf_price + } + + if( db.qfang_esf_price && db.qfang_esf_price>3000){ + prices << db.qfang_esf_price + } + + if(db.fangtan_esf_price && db.fangtan_esf_price>3000){ + prices << db.fangtan_esf_price + } + + if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){ + prices << db.fangduoduo_esf_price + } + + if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + prices << db.get("5i5j_esf_price") + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 2f73eda31..f67de27ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1") + def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 1bd63cbeb..1691b8656 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -22,25 +22,24 @@ class ExportOracleData { def run() { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); - def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // //不会变 -// exportEvaluationRateData() -// exportBigdataPoiData() -// exportBigdataDistrictData() + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() // -// exportGuidedPrice()//随人工改变 -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 -// -// //处理小区情况 -// exportBigdataCommunity() -// exportCommunityStaging() -// exportCommunityBuileType() + exportGuidedPrice()//随人工改变 + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 -// exportThirdPartyPrice2(lastMonth) -// exportBigdataForsale("2021-12-19") -// exportBigdataDeal("2021-12-01") +// //处理小区情况 + exportBigdataCommunity() + exportCommunityStaging() + exportCommunityBuileType() + + exportThirdPartyPrice2(lastMonth) + exportBigdataForsale("2022-01-19") + exportBigdataDeal("2022-01-01") //计算结果相关 @@ -56,11 +55,11 @@ class ExportOracleData { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) - def curDate = curMonth.toString()+"28" - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// def curDate = curMonth.toString()+"28" +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); //ok文件 - exportLastFile() +// exportLastFile() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index 84f2232df..42ad78607 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -23,9 +23,10 @@ class LoadGuide { // saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx") - savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") + savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") } def match(def filename){ @@ -48,15 +49,15 @@ class LoadGuide { def datas = ExcelUtils.read(filePath,0, 1) def rets = [] for(def d : datas){ - if(d['E']){ + if(d['D']){ def o = new BasicDBObject() - o.city = d['E'] + o.city = d['D'] o.region = d['A'] o.name = d['B'] o.avg_price = d['C'].toString().toDouble().toInteger() - o.match_city = d['E'] - o.match_region = d['F'] - o.match_name = d['G'] + o.match_city = d['D'] + o.match_region = d['E'] + o.match_name = d['F'] o.source_type = 1 //1:成交参考价 2:参考价 o.c_date = new Date() rets << o diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 53a43fb77..dc639ef71 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") -def testMap = findCaluateCityPPE(sqlInstance236, 202201) -ExcelUtils.modify(fileName,{d-> - if(d['C']){ - def oldcount = dealMap.get(d['C']) - d['T'] = oldcount - def casecount = testMap.get(d['C']) - d['U'] = casecount - println d - } -},0) +//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202201) +//ExcelUtils.modify(fileName,{d-> +// if(d['C']){ +// def oldcount = dealMap.get(d['C']) +// d['T'] = oldcount +// def casecount = testMap.get(d['C']) +// d['U'] = casecount +// println d +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -270,20 +270,25 @@ ExcelUtils.modify(fileName,{d-> // } //},0) -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d-> -// if(d['A']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['A'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -//},0) +def index = 0 +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> + def flag = d['M'] + if(flag==0.0){ + if(d['D']){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['D'])) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + index++ + println index + } + } + +},0) + //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ // def region = d['B'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 1e18741df..c6eec9d7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -36,7 +36,8 @@ public class LianjiaCrawlByChrome { def task = new LianjiaCrawlByChrome(); // task.run() // task.run1() - task.addDealRegion(); +// task.addDealRegion(); + task.standardRegion() } def run1(){ @@ -105,6 +106,28 @@ public class LianjiaCrawlByChrome { } + def standardRegion(){ + + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) + def dbcursor = DEAL_PRICE_COLL.find(q) + for(def d : dbcursor){ + if(d.region){ + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion && newRegion!=d.region){ + def query = new BasicDBObject() + query.put("city", d.city) + query.put("region", d.region) + def data = new BasicDBObject() + data.put("region", newRegion) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + + } + } + def addDealRegion(){ def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") lastMonth = lastMonth +"-01" @@ -132,6 +155,7 @@ public class LianjiaCrawlByChrome { def data = new BasicDBObject() data.put("region", dist.region) + println m.city+ " "+m.name +" "+dist.region def ds = DEAL_PRICE_COLL.count(query) DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) }