From b734dfc285522ab8f4c022ec3988fb945762f79f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Nov 2022 13:14:51 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 16 +-- .../ccbevaluation/ExportOracleDataTest.groovy | 8 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 9 +- .../ccbevaluation/data/AddressCut.groovy | 97 ++++++++++++++++++- .../ccbevaluation/data/CcbDataStat.groovy | 18 +++- .../script/gj/es/EsMonthHouseStat.groovy | 4 +- 7 files changed, 135 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 5b0edbae1..53c7c83c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -67,7 +67,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { def index = 0 - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args) statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ @@ -75,8 +75,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221019); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) @@ -477,8 +477,14 @@ class BigdataForsaleBatch extends BaseBatch{ println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type continue; } + + def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) + if(vallaFlag){ + continue; + } } + if(!seawater.total_price){ continue; } @@ -538,10 +544,6 @@ class BigdataForsaleBatch extends BaseBatch{ if(floor && floor.toInteger()<1){ floor = null } - def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) - if(vallaFlag){ - continue; - } def rowMap = [:]; if(cat == "rent"){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 431fc02c2..c4d5579f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -976,9 +976,9 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); @@ -1007,9 +1007,9 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 6981c7149..bf34601ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 6b1009e6c..02afd29da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202107 - def endDate = 202109 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -503,7 +503,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def q = new BasicDBObject() q.put("city", city) + //q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index 19bf6561b..4c66509ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -7,8 +7,12 @@ import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.ResultSet import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -16,13 +20,79 @@ import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong class AddressCut extends BaseBatch{ + + + final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } static main(args) { runBatch(new AddressCut(), args); } @Override public void process(Object args) { - sameAddr() +// sameAddr() //相同地址小区 +// roadMatch() + addAnjukeRoad() + + } + + def addAnjukeRoad(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> + def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") + if(roads){ + d['O'] = roads[0] + } + }); + } + + def findThirdPartyPrice(city, region, name, source){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", source) + def dbcursor = thirdPartyPriceColl.find(q).sort(new BasicDBObject("s_date", -1)).limit(10) + def roads = [] + for(def d : dbcursor){ + if(d.address && !roads.contains(d.address)){ + roads << d.address + } + } + return roads + } + + def roadMatch(){ + def cityRoadsMap = queryPoiMap() + println "道路加载完成....." + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> + def roads = cityRoadsMap.get(d['A']) +// if (roads && roads.contains(d['E'])) { +// d['L'] = "是" +// println d['A'] + " " + d['E'] +// } + + if(roads){ + for(def road : roads){ + if(!d['L'] && d['E'] && d['E'].toString().contains(road)){ + d['L'] = "中间包含" + d['M'] = road + println d['A'] + " " + d['E'] + break; + } + } + } + }) + + } def sameAddr() { @@ -143,4 +213,29 @@ class AddressCut extends BaseBatch{ latch.await() return rows; } + + def queryPoiMap(){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,name,type FROM amap_poi WHERE 1=1") +// sqlBuffer.append(" and cityname = '" + city + "'") +// sqlBuffer.append(" and name = '" + name + "'") + sqlBuffer.append(" and type like '%道路名%'") +// sqlBuffer.append(" and MATCH(name,'${name}')") +// sqlBuffer.append(" and source = '高德' order by score() desc") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def cityMap = [:] + while (rs.next()) { + def city = rs.getString("cityname"); + def road = rs.getString("name") + def roads = cityMap.get(city) + if(!roads){ + roads = new HashSet() + } + roads << road + cityMap.put(city, roads) + + } + return cityMap; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy index 1aa5a391f..d7be2d646 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation.data import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils @@ -19,7 +20,22 @@ class CcbDataStat { static main(args) { def task = new CcbDataStat() // task.run() - task.tradeRun() +// task.tradeRun() + task.regionDistStat() + } + + def regionDistStat(){ + def citys = CcbCommonUtils.citys_252 + def rows = [] + for(def c : citys){ + def regions = CommonDatautils.getRegionListByCity(c) + for(def r : regions){ + def count = CommonDatautils.findRegionDistrictCount("district",c, r) + rows << [c, r, count] + println c +" "+r +" "+count + } + } + ExcelUtils.write("d:\\数据\\建行\\区域小区量统计.xlsx", rows, "统计") } def tradeRun(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index 3d79e39fb..9c68829cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,11 +19,11 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() -// task.houseRun() + task.houseRun() // task.statHouseUpdateCount() - task.statDistHouse() +// task.statDistHouse() } def statDistHouse(){