Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+2
-2
@@ -440,7 +440,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
|
||||
println "原始:"+dbsour.count()
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
@@ -450,7 +450,7 @@ class CcbCommonUtils {
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
println "小区量:"+rets.size()
|
||||
println "[${city}] 小区量:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
+7
-7
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
// task.updatePrice()
|
||||
// task.addFjwPrice()
|
||||
task.addFjwPrice()
|
||||
// task.addCcbPrice()
|
||||
|
||||
// task.addCcbPriceFromGuided()
|
||||
@@ -44,7 +44,7 @@ class FjwPricePkCcbPrice {
|
||||
|
||||
// task.addCcbPrice2()
|
||||
// task.addCcbPrice3()
|
||||
task.standRegion()
|
||||
// task.standRegion()
|
||||
}
|
||||
|
||||
def addCcbPrice2(){
|
||||
@@ -202,13 +202,13 @@ class FjwPricePkCcbPrice {
|
||||
}
|
||||
|
||||
def addFjwPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
|
||||
if(!d['M']){
|
||||
def dist = CommonDatautils.findDistrictById(d['D'])
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(dist){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['M'] = dbo.avg_price
|
||||
d['K'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+348
-13
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
@@ -28,25 +29,41 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
|
||||
def curRunDate = null
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
public static def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCitypriceBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
cityRun(citys, startDate, endDate, cityCodeDataMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeDataMap)
|
||||
districtRun(citys, startDate, endDate, cityCodeDataMap)
|
||||
def curDate = new Date()
|
||||
def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args)
|
||||
if(args){
|
||||
def dateStr = args[0]
|
||||
curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01
|
||||
}
|
||||
|
||||
println "args 参数:"+args.toString() +" 城市数量:"+citys.size()+" curDate: "+ curDate
|
||||
|
||||
curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger();
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
def cityRun(citys, startDate, endDate, cityCodeDataMap){
|
||||
@@ -177,11 +194,18 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def districtRun(citys, startDate, endDate, cityCodeDataMap){
|
||||
def districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap){
|
||||
def rets = []
|
||||
def allDistricts = []
|
||||
def allPriceMap = [:]
|
||||
def allRentMap = [:]
|
||||
def allDealCountMap = [:]
|
||||
def allCityRankMap = [:]
|
||||
def allRegionRankMap = [:]
|
||||
def allDistCityPreMap = [:]
|
||||
def allDistRegionPreMap = [:]
|
||||
def allLastMonthMap = [:]
|
||||
def allLastYearMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
|
||||
allDistricts.addAll(districts)
|
||||
@@ -192,9 +216,36 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
|
||||
allRentMap = allRentMap + rentMap
|
||||
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
|
||||
allDealCountMap = allDealCountMap + dealCountMap
|
||||
|
||||
|
||||
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
|
||||
|
||||
def cityRankMap = findCommunityCityRank(datas)
|
||||
allCityRankMap = allCityRankMap + cityRankMap
|
||||
|
||||
def regionRankMap = findCommunityRegionRank(datas)
|
||||
allRegionRankMap = allRegionRankMap + regionRankMap
|
||||
|
||||
def cityPrice = findCcbCityPrice(cityCode, startDate.toString().substring(0,6))
|
||||
def regionPriceMap = findCcbRegionPrice(cityCode, startDate.toString().substring(0,6))
|
||||
|
||||
def cityPreMap = districtCityPrictRate(cityPrice, datas)
|
||||
allDistCityPreMap = allDistCityPreMap + cityPreMap
|
||||
|
||||
def regionPreMap = districtRegionPriceRate(regionPriceMap, datas)
|
||||
allDistRegionPreMap = allDistRegionPreMap + regionPreMap
|
||||
|
||||
def lastMonthMap = findDistQuotations(cityCode, dealStartDate.replace("-","").substring(0,6))
|
||||
allLastMonthMap = allLastMonthMap + lastMonthMap
|
||||
|
||||
def lastYearMonthMap = findDistQuotations(cityCode, lastYearDate.toString().substring(0,6))
|
||||
allLastYearMap = allLastYearMap + lastYearMonthMap
|
||||
}
|
||||
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap)
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
if(cityLists){
|
||||
rets.addAll(cityLists)
|
||||
}
|
||||
@@ -212,7 +263,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def totalCount = new AtomicLong()
|
||||
def distCount = districts.size()
|
||||
final CountDownLatch latch = new CountDownLatch(distCount)
|
||||
@@ -220,11 +271,12 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
if(row){
|
||||
rets << row
|
||||
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
|
||||
@@ -271,7 +323,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def sellDataType = 1
|
||||
def rentDataType = 1
|
||||
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
@@ -400,6 +452,160 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
row << 1
|
||||
row << 'FJW'
|
||||
|
||||
//成交样本量,成交样本量同比, 成交样本量环比
|
||||
if(dealCountMap){
|
||||
def dealCount = dealCountMap.get(dist._id.toString())
|
||||
row << dealCount?dealCount:''
|
||||
def dealMarkup = ''
|
||||
def dealYoy = ''
|
||||
if(allLastMonthMap){
|
||||
def dealMarkupMap = allLastMonthMap.get(dist._id.toString())
|
||||
try{
|
||||
if(dealCount && dealMarkupMap && dealMarkupMap.deal_count && dealMarkupMap.deal_count>0){
|
||||
def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count
|
||||
dealMarkup = DataHandleUtils.getDoubleRound(2, markup)
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
}
|
||||
if(allLastYearMap){
|
||||
def dealYoyMap = allLastYearMap.get(dist._id.toString())
|
||||
if(dealCount && dealYoyMap && dealYoyMap.deal_count && dealYoyMap.deal_count>0){
|
||||
def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count
|
||||
dealYoy = DataHandleUtils.getDoubleRound(2, markup)
|
||||
}
|
||||
}
|
||||
row << dealYoy
|
||||
row << dealMarkup
|
||||
|
||||
}else{
|
||||
row << ''
|
||||
row << ''
|
||||
row << ''
|
||||
}
|
||||
|
||||
//出售平均单价的同比,环比
|
||||
if(sell_avg_price){
|
||||
def markup = ''
|
||||
if(allLastMonthMap){
|
||||
def dataMap = allLastMonthMap.get(dist._id.toString())
|
||||
if(dataMap && dataMap.saleprice){
|
||||
markup = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice
|
||||
markup = DataHandleUtils.getDoubleRound(2, markup)
|
||||
}
|
||||
}
|
||||
def yoy = ''
|
||||
if(allLastYearMap){
|
||||
def dataMap = allLastYearMap.get(dist._id.toString())
|
||||
if(dataMap && dataMap.saleprice){
|
||||
yoy = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice
|
||||
yoy = DataHandleUtils.getDoubleRound(2, yoy)
|
||||
}
|
||||
}
|
||||
row << yoy
|
||||
row << markup
|
||||
}else{
|
||||
row << ''
|
||||
row << ''
|
||||
}
|
||||
|
||||
//出售样本量同比,环比
|
||||
if(sell_count){
|
||||
def markup = ''
|
||||
if(allLastMonthMap){
|
||||
def dataMap = allLastMonthMap.get(dist._id.toString())
|
||||
if(dataMap && dataMap.salecount){
|
||||
markup = (sell_count - dataMap.salecount)*100/dataMap.salecount
|
||||
markup = DataHandleUtils.getDoubleRound(2, markup)
|
||||
}
|
||||
}
|
||||
def yoy = ''
|
||||
if(allLastYearMap){
|
||||
def dataMap = allLastYearMap.get(dist._id.toString())
|
||||
if(dataMap && dataMap.salecount){
|
||||
yoy = (sell_count - dataMap.salecount)*100/dataMap.salecount
|
||||
yoy = DataHandleUtils.getDoubleRound(2, yoy)
|
||||
}
|
||||
}
|
||||
row << yoy
|
||||
row << markup
|
||||
}else{
|
||||
row << ''
|
||||
row << ''
|
||||
}
|
||||
|
||||
//出租房源量同比,环比
|
||||
if(rent_count){
|
||||
def markup = ''
|
||||
if(allLastMonthMap){
|
||||
def dataMap = allLastMonthMap.get(dist._id.toString())
|
||||
if(dataMap && dataMap.salecount && dataMap.rentcount>0){
|
||||
markup = (rent_count - dataMap.rentcount)*100/dataMap.rentcount
|
||||
markup = DataHandleUtils.getDoubleRound(2, markup)
|
||||
}
|
||||
}
|
||||
def yoy = ''
|
||||
if(allLastYearMap){
|
||||
def dataMap = allLastYearMap.get(dist._id.toString())
|
||||
if(dataMap && dataMap.salecount && dataMap.rentcount>0){
|
||||
yoy = (rent_count - dataMap.rentcount)*100/dataMap.rentcount
|
||||
yoy = DataHandleUtils.getDoubleRound(2, yoy)
|
||||
}
|
||||
}
|
||||
row << yoy
|
||||
row << markup
|
||||
}else{
|
||||
row << ''
|
||||
row << ''
|
||||
}
|
||||
//小区价格城市排名
|
||||
if(allCityRankMap){
|
||||
def cityRank = allCityRankMap.get(dist._id.toString())
|
||||
if(cityRank){
|
||||
row << cityRank
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
//小区价格区域排名
|
||||
if(allRegionRankMap){
|
||||
def regionRank = allRegionRankMap.get(dist._id.toString())
|
||||
if(regionRank){
|
||||
row << regionRank
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
//高于区县平均价比率
|
||||
if(allDistRegionPreMap){
|
||||
def regionRadio = allDistRegionPreMap.get(dist._id.toString())
|
||||
if(regionRadio){
|
||||
row << regionRadio
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
|
||||
//高于城市平均价比率
|
||||
if(allDistCityPreMap){
|
||||
def cityRadio = allDistCityPreMap.get(dist._id.toString())
|
||||
if(cityRadio){
|
||||
row << cityRadio
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
return row
|
||||
}
|
||||
return null
|
||||
@@ -723,6 +929,135 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def findDealDatesByCity(cityCode, startDate, endDate){
|
||||
def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id";
|
||||
def rMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
rMap.put(r.gujia_comm_id, r.count)
|
||||
})
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def findCommunityCityRank(datas){
|
||||
def values = []
|
||||
for(def m : datas){
|
||||
values << m.avg_price
|
||||
}
|
||||
values = values.sort().reverse()
|
||||
|
||||
def resMap = [:]
|
||||
def total = values.size()
|
||||
for(def m : datas){
|
||||
// def index = 0
|
||||
// for(def v : values){
|
||||
// index++
|
||||
// if(m.avg_price == v){
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
def index = values.indexOf(m.avg_price)+1
|
||||
resMap.put(m.comm_id, index+"|"+total)
|
||||
}
|
||||
return resMap
|
||||
}
|
||||
|
||||
def findCommunityRegionRank(datas){
|
||||
|
||||
def regionMap = [:]
|
||||
for(def m : datas){
|
||||
def prices = regionMap.get(m.region)
|
||||
if(!prices){
|
||||
prices = []
|
||||
regionMap.put(m.region, prices)
|
||||
}
|
||||
prices << m.avg_price
|
||||
prices.sort().reverse()
|
||||
}
|
||||
|
||||
|
||||
def resMap = [:]
|
||||
for(def m : datas){
|
||||
def prices = regionMap.get(m.region)
|
||||
def total = prices.size()
|
||||
def index = prices.indexOf(m.avg_price)+1
|
||||
// for(def v : prices){
|
||||
// index++
|
||||
// if(m.avg_price == v){
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
resMap.put(m.comm_id, index+"|"+total)
|
||||
}
|
||||
return resMap
|
||||
}
|
||||
|
||||
def findCommunityPriceByCity(cityCode, month){
|
||||
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def datas = []
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
datas << ["comm_id":r.gujia_bds_id, "avg_price":r.gujia_bds_clc_avprc, "region":r.gujia_admndiv_nm]
|
||||
})
|
||||
return datas;
|
||||
}
|
||||
|
||||
def findCcbCityPrice(cityCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def avgPrice = null
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
avgPrice = r.gujia_bds_clc_avprc
|
||||
})
|
||||
return avgPrice;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findCcbRegionPrice(cityCode, month){
|
||||
def sql = "SELECT gxda_admndiv_nm,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def regionPriceMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
regionPriceMap.put(r.gxda_admndiv_nm, r.gujia_bds_clc_avprc)
|
||||
})
|
||||
return regionPriceMap;
|
||||
}
|
||||
|
||||
|
||||
def districtCityPrictRate(cityPrice, datas){
|
||||
def distCityRateMap = [:]
|
||||
if(cityPrice){
|
||||
for(def m : datas){
|
||||
def val = (m.avg_price-cityPrice)*100/cityPrice
|
||||
distCityRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val))
|
||||
}
|
||||
}
|
||||
return distCityRateMap;
|
||||
}
|
||||
|
||||
|
||||
def districtRegionPriceRate(regionPriceMap, datas){
|
||||
def distRegionRateMap = [:]
|
||||
if(regionPriceMap){
|
||||
for(def m : datas){
|
||||
def regionAvgPrice = regionPriceMap.get(m.region)
|
||||
if(regionAvgPrice){
|
||||
def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice
|
||||
distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val))
|
||||
}else{
|
||||
println m.region+" 无区域均价"
|
||||
}
|
||||
}
|
||||
}
|
||||
return distRegionRateMap;
|
||||
}
|
||||
|
||||
def findDistQuotations(cityCode, month){
|
||||
def sql = "select commid, deal_count, saleprice,salecount,rentcount from bigdata_commprice_fjw where ymid=${month} and cityid=${cityCode}";
|
||||
def markupMap = [:]
|
||||
oracleSqlInstance.eachRow(sql, { r ->
|
||||
markupMap.put(r.commid, ["deal_count":r.DEAL_COUNT, "saleprice":r.SALEPRICE,"salecount":r.SALECOUNT,"rentcount":r.RENTCOUNT])
|
||||
})
|
||||
return markupMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+15
-2
@@ -85,7 +85,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportPgBigdataModelingBase(lastMonth)
|
||||
|
||||
//2月8号 临时导出苏州一年的数据
|
||||
exportPgBigdataForsale("2023-01-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
}
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
@@ -1669,7 +1669,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
|
||||
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE,DEAL_COUNT,DEAL_COUNT_YOY,DEAL_COUNT_MOM,NETLIST_PRICE_YOY,NETLIST_PRICE_MOM,NETLIST_COUNT_YOY,NETLIST_COUNT_MOM,RENT_COUNT_YOY,RENT_COUNT_MOM,CITY_RANK,AREA_RANK,DISTRICT_PRICE_THAN,CITY_PRICE_THAN from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
@@ -1697,6 +1697,19 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.RDATERANGE
|
||||
data << r.DATATYPE
|
||||
data << r.SOURCECODE
|
||||
data << r.DEAL_COUNT
|
||||
data << r.DEAL_COUNT_YOY
|
||||
data << r.DEAL_COUNT_MOM
|
||||
data << r.NETLIST_PRICE_YOY
|
||||
data << r.NETLIST_PRICE_MOM
|
||||
data << r.NETLIST_COUNT_YOY
|
||||
data << r.NETLIST_COUNT_MOM
|
||||
data << r.RENT_COUNT_YOY
|
||||
data << r.RENT_COUNT_MOM
|
||||
data << r.CITY_RANK
|
||||
data << r.AREA_RANK
|
||||
data << r.DISTRICT_PRICE_THAN
|
||||
data << r.CITY_PRICE_THAN
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
+158
-28
@@ -5,6 +5,7 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -14,6 +15,9 @@ import groovy.sql.Sql
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
@@ -26,36 +30,37 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||
//按天导(每天一个文件夹)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
||||
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
||||
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
|
||||
// if(curDate.toString().endsWith("10")){
|
||||
|
||||
exportEvaluateRate()
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
|
||||
|
||||
exportBigdataCommunityGJRH()//
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
exportBigdataCommhouseuseGJ()//
|
||||
exportBigdataPositionGJ()//
|
||||
exportBigdataDistrictFJW()//
|
||||
exportBigdataAliasFJW()//
|
||||
|
||||
|
||||
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
// exportEvaluateRate()
|
||||
// exportBigdataBuildprojFjw()
|
||||
// exportBigdataCommlayoutpicFjw()
|
||||
//
|
||||
//
|
||||
// exportBigdataCommunityGJRH()/
|
||||
// exportBigdataBuildingFJW()//
|
||||
// exportBigdataCommfeeGj()//
|
||||
//
|
||||
// exportBigdataCommbuildstruGJ()//
|
||||
// exportBigdataCommhousetypeGJ()//
|
||||
// exportBigdataCommbuildyearGj()//
|
||||
// exportBigdataCommhouseuseGJ()//
|
||||
// exportBigdataPositionGJ()//
|
||||
// exportBigdataDistrictFJW()//
|
||||
// exportBigdataAliasFJW()//
|
||||
//
|
||||
//
|
||||
// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
|
||||
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
// exportBigdataCityPrice(queryDate)
|
||||
// exportBigdataDistPrice(queryDate)
|
||||
// exportBigdataCommPrice(queryDate)
|
||||
|
||||
exportInOk()
|
||||
|
||||
@@ -528,6 +533,50 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def exportPgBigdataDealFJW(curDate, dealdate, new_dists){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
|
||||
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
|
||||
def allRows = []
|
||||
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
if(curDate.toString().endsWith("10")){
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
|
||||
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealdate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
|
||||
def rets1 = dealData(sql1)
|
||||
|
||||
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
|
||||
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
|
||||
def rets2 = dealData(sql2)
|
||||
allRows.addAll(rets1)
|
||||
allRows.addAll(rets2)
|
||||
}else{
|
||||
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
|
||||
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";// and txn_dt>='${dealdate}'
|
||||
allRows.addAll(dealData(sql))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
|
||||
def createDeal(sql){
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
@@ -607,6 +656,87 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
return allRows;
|
||||
}
|
||||
|
||||
|
||||
def dealData(def sql){
|
||||
def allRows = []
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_bds_mdl_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_str
|
||||
data << r.gujia_hsno
|
||||
data << r.gujia_comm_id
|
||||
data << r.comm_nm
|
||||
data << r.gujia_bld_id
|
||||
data << r.bld_id
|
||||
data << r.gujia_bld_nm
|
||||
data << r.gujia_unit_id
|
||||
data << r.unit_id
|
||||
data << r.gujia_unit_nm
|
||||
data << r.gujia_acc_id
|
||||
data << r.gujia_dtl_adr
|
||||
data << r.gujia_bds_bld_yr_rc
|
||||
data << r.gujia_own_val
|
||||
data << r.gujia_own_yrlmt_val
|
||||
data << r.gujia_prd_val
|
||||
data << r.gujia_bds_wbt_floor
|
||||
data << r.gujia_lyr_tp_val
|
||||
data << r.gujiabdstot_floor_num
|
||||
if(r.gujia_bld_tp_val){
|
||||
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
|
||||
data << r.gujia_bld_stc
|
||||
data << r.guiabconstructionarea
|
||||
data << r.gujiabdsset_innr_area
|
||||
data << r.gujia_acc_tp_val
|
||||
data << r.gujia_bds_room_num
|
||||
data << r.gujia_bds_dept_num
|
||||
data << r.gujia_bds_ktch_num
|
||||
data << r.gujia_bds_hlt_btw_num
|
||||
data << r.gujia_hs_use
|
||||
data << r.gujia_indr_stc_val
|
||||
data << r.gujia_twrd_val
|
||||
data << r.gujia_dcrt_sttn
|
||||
data << r.gujia_ases_bt_ln_num
|
||||
data << r.gujia_bds_trdtckt_prc
|
||||
data << r.gujia_bds_mdl_tot_prc
|
||||
data << r.gujia_bds_dcl_tot_prc
|
||||
data << r.gujiabds_ases_tot_prc
|
||||
data << r.gujiasigntonlinectrid
|
||||
data << r.txn_dt
|
||||
data << r.gujia_txn_tp_val
|
||||
data << r.gujia_sell_pymtmod
|
||||
data << r.gujia_bds_mdl_cyc
|
||||
data << r.gujiawthrext_lift_ind
|
||||
data << r.gujiabdslndfgrngvarea
|
||||
data << r.gujiabdslutbl_gv_area
|
||||
data << r.gujiabdssuntblgv_area
|
||||
data << r.gujiabdsbayindowgarea
|
||||
data << r.gujiabdstowerfrgvarea
|
||||
data << r.gujia_bds_grg_gv_area
|
||||
data << r.gujiabdsundgdrmgvarea
|
||||
data << r.gujiabds_othr_gv_area
|
||||
data << r.gujia_hs_src_dtsrc
|
||||
data << r.intr_co_id
|
||||
data << r.intr_co_nm
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
// data << r.gujia_bds_mkdam
|
||||
data << r.gujia_bds_data_tm_val
|
||||
allRows << data
|
||||
});
|
||||
return allRows
|
||||
}
|
||||
|
||||
def exportAllBigdataDealFJW(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
@@ -760,7 +890,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
|
||||
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";//
|
||||
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}'";//
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -807,7 +937,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate}
|
||||
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}'";// and YMID=${queryDate}
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
@@ -854,7 +984,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate}
|
||||
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID>=202210";// and YMID=${queryDate}
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
class UpdateCommPrice {
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
public static def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
static main(args) {
|
||||
def task = new UpdateCommPrice()
|
||||
task.run(args)
|
||||
}
|
||||
|
||||
def run(Object args){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def curDate = new Date()
|
||||
if(args){
|
||||
def dateStr = args[0]
|
||||
curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01
|
||||
}
|
||||
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger();
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
process(citys, dealStartDate, startDate, curRunDate)
|
||||
|
||||
}
|
||||
|
||||
def process(citys, dealStartDate, startDate, curRunDate){
|
||||
def rets = []
|
||||
def allDistricts = []
|
||||
def allDealCountMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias"]);
|
||||
allDistricts.addAll(districts)
|
||||
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
|
||||
allDealCountMap = allDealCountMap + dealCountMap
|
||||
}
|
||||
|
||||
for(def i=0;i<allDistricts.size();i++) {
|
||||
final def dist = allDistricts[i];
|
||||
def dealCount = allDealCountMap.get(dist._id.toString())
|
||||
if(dealCount){
|
||||
updateDealCount(dist._id.toString(), dealCount, curRunDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateDealCount(def dist_id, count, month){
|
||||
def sql = "update bigdata_commprice_fjw set deal_count=${count} where ymid=${month} and commid='${dist_id}'"
|
||||
oracleSqlInstance.execute(sql)
|
||||
}
|
||||
|
||||
def findDealDatesByCity(cityCode, startDate, endDate){
|
||||
def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id";
|
||||
def rMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
rMap.put(r.gujia_comm_id, r.count)
|
||||
})
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def calcMonthlyPrice(){
|
||||
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
|
||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
||||
@@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
index++
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList()
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
}else{
|
||||
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
|
||||
}
|
||||
|
||||
+41
-2
@@ -18,13 +18,13 @@ class AddAvgpriceBatch {
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static {
|
||||
citys = CommonDatautils.findAllCity()
|
||||
citys = ["徐州"]//CommonDatautils.findAllCity()
|
||||
println "加载城市完成:"+citys.size()
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
def task = new AddAvgpriceBatch();
|
||||
task.run();
|
||||
task.run1();
|
||||
}
|
||||
def run(){
|
||||
|
||||
@@ -63,6 +63,45 @@ class AddAvgpriceBatch {
|
||||
|
||||
|
||||
|
||||
def run1(){
|
||||
|
||||
def totalCount = 0;
|
||||
citys.each{city->
|
||||
if(city!="全国"){
|
||||
def districtKeysMap = CommonDatautils.findAllDistrictKeys(city, "district")
|
||||
println "------------------------------------------------------------"
|
||||
println "${city}小区量:"+districtKeysMap.size()
|
||||
if(districtKeysMap){
|
||||
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26"))
|
||||
println "【2023-03-26】共计均价小区:"+lastWeekendStats.size()
|
||||
//库中小区没有计算过均价的小区
|
||||
def districtKeys = districtKeysMap.keySet();
|
||||
def rets1 = findNoCaluteDistrict(districtKeys, lastWeekendStats);
|
||||
println "【2023-03-26】共 "+rets1.size()+" 个小区无均价"
|
||||
if(rets1){
|
||||
for(def disKey : rets1){
|
||||
def array = disKey.split("##");
|
||||
def estateType2 = districtKeysMap.get(disKey)
|
||||
//发重算均价命令
|
||||
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println "共补:"+totalCount
|
||||
}
|
||||
|
||||
|
||||
def addPrice(city, region, name, estateType2,startDate, endDate){
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHavePrice(lastAgoWeekendStats, rets){
|
||||
def disKeys = [];
|
||||
for(def key : rets){
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.Document
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
|
||||
def rMap = [:]
|
||||
def index = 0
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
|
||||
index++
|
||||
def day = doc.c_date.format("yyyy-MM-dd")
|
||||
def count = rMap.get(day)
|
||||
if (!count) {
|
||||
rMap.put(day, 1)
|
||||
} else {
|
||||
rMap.put(day, count + 1)
|
||||
}
|
||||
if(index%10000==0){
|
||||
println index+" "+new Date().toString()
|
||||
}
|
||||
}
|
||||
|
||||
rMap.each {k,val->
|
||||
println k+" "+val
|
||||
}
|
||||
|
||||
|
||||
+696
-25
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
@@ -11,26 +12,71 @@ import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AnjukeDistrictMatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDistrictMatch();
|
||||
// task.run();
|
||||
task.matchCommunity()
|
||||
// task.runCsv();
|
||||
// task.updateCredit()
|
||||
// task.matchCommunity()//查询es小区数据
|
||||
// task.run1()
|
||||
// task.updateField()
|
||||
// task.updateGreenRateField()//根据第三方数据填补库中字段
|
||||
// task.updteGreenRateFieldByCsv()
|
||||
// task.fenQiRun()
|
||||
|
||||
// task.findAliasDatas("淮安","水沐未来印象花园")
|
||||
// task.findAliasDatas("淮安","水沐未来印象花园")//模糊查找Es别名数据
|
||||
// task.repeatDistByAlias()//根据第三方网站别名查找重复数据
|
||||
|
||||
// task.matchCommunityByCsv()
|
||||
// task.updteBuildAreaFieldByCsv()
|
||||
// task.updadteAllAreaFieldByCsv()
|
||||
// task.updatePlotRateFieldByCsv()
|
||||
task.updateFinishDateFieldByCsv()
|
||||
}
|
||||
|
||||
def updateCredit(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv")
|
||||
def results = []
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
if(row[24] && row[24].contains("接口")){
|
||||
def inputName = row[2]
|
||||
def matchName = row[27]
|
||||
if(inputName.toString().endsWith("小区") || inputName.toString().endsWith("院")){
|
||||
inputName = inputName.replace("小区","").replace("院","")
|
||||
}
|
||||
if(matchName.toString().endsWith("小区") || matchName.toString().endsWith("院")){
|
||||
matchName = matchName.replace("小区","").replace("院","")
|
||||
}
|
||||
if(inputName == matchName && row[1]==row[26]){
|
||||
row[30] = 1
|
||||
}
|
||||
}
|
||||
|
||||
results << row
|
||||
}
|
||||
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv",null,"GBK")
|
||||
batchCreateFile(datas, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-2.csv",null,"GBK")
|
||||
}
|
||||
|
||||
|
||||
@@ -96,7 +142,7 @@ class AnjukeDistrictMatch {
|
||||
}
|
||||
|
||||
|
||||
def updateField(){
|
||||
def updateGreenRateField(){
|
||||
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(row['Q'] && !row['Q'].contains("接口")){
|
||||
@@ -155,18 +201,547 @@ class AnjukeDistrictMatch {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def matchCommunityByCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv")
|
||||
def map = [:]
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
map.put(row[0]+"#"+row[1]+"#"+row[2], row[24]+"#"+row[25]+"#"+row[26]+"#"+row[27]+"#"+row[28])
|
||||
}
|
||||
|
||||
def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间.csv")
|
||||
def lastRets = []
|
||||
for(def i=0;i<datas1.size();i++) {
|
||||
final def data = datas1[i]
|
||||
def row = data as List
|
||||
def resultStr = map.get(row[0]+"#"+row[1]+"#"+row[2])
|
||||
if(resultStr){
|
||||
def arrs = resultStr.split("#")
|
||||
if(arrs){
|
||||
row << arrs[0]
|
||||
if(arrs.size()>1){
|
||||
row << arrs[1]
|
||||
}
|
||||
if(arrs.size()>2){
|
||||
row << arrs[2]
|
||||
}
|
||||
if(arrs.size()>3){
|
||||
row << arrs[3]
|
||||
}
|
||||
if(arrs.size()>4){
|
||||
row << arrs[4]
|
||||
}
|
||||
}
|
||||
}
|
||||
lastRets << row
|
||||
}
|
||||
batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间(匹配).csv",null,"GBK")
|
||||
}
|
||||
|
||||
def updteGreenRateFieldByCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\测试.csv")
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
if(row[24] && (row[24]=="名称匹配"||row[24]=="映射别名匹配" || (row[24].contains("接口") && row[28] && row[28].toString().isNumber() && row[28].toString().toDouble()>=0.9))){
|
||||
def dist = CommonDatautils.findSeaweedById(row[25], ["virescence_percent"])
|
||||
if(dist && !dist.virescence_percent){
|
||||
def val = row[12]
|
||||
|
||||
if(!val){
|
||||
val = row[13]
|
||||
}
|
||||
if(!val){
|
||||
val = row[14]
|
||||
}
|
||||
if(!val){
|
||||
val = row[11]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[21]
|
||||
}
|
||||
if(!val){
|
||||
val = row[22]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[6]
|
||||
}
|
||||
if(!val){
|
||||
val = row[10]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[9]
|
||||
}
|
||||
if(!val){
|
||||
val = row[5]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[16]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[15]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[23]
|
||||
}
|
||||
if(!val){
|
||||
val = row[19]
|
||||
}
|
||||
if(!val){
|
||||
val = row[17]
|
||||
}
|
||||
if(!val){
|
||||
val = row[8]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[18]
|
||||
}
|
||||
if(!val){
|
||||
val = row[20]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[3]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[4]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[7]
|
||||
}
|
||||
|
||||
|
||||
if(val){
|
||||
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
|
||||
val = val.toString().toDouble().toInteger()
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("virescence_percent", val.toString())
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(dist._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
println row[25] +" "+val.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updteBuildAreaFieldByCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站建筑面积(匹配).csv")
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
if(row[16] && (row[16]=="名称匹配"||row[16]=="映射别名匹配" || (row[16].contains("接口") && row[20] && row[20].toString().isNumber() && row[20].toString().toDouble()>=0.9))){
|
||||
def dist = CommonDatautils.findSeaweedById(row[17], ["build_area"])
|
||||
if(dist && !dist.build_area){
|
||||
def val = row[8]
|
||||
|
||||
if(!val){
|
||||
val = row[7]
|
||||
}
|
||||
if(!val){
|
||||
val = row[9]
|
||||
}
|
||||
if(!val){
|
||||
val = row[14]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[15]
|
||||
}
|
||||
if(!val){
|
||||
val = row[12]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[5]
|
||||
}
|
||||
if(!val){
|
||||
val = row[6]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[3]
|
||||
}
|
||||
if(!val){
|
||||
val = row[13]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[4]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[10]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[11]
|
||||
}
|
||||
|
||||
|
||||
if(val){
|
||||
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
|
||||
val = val.toString().toDouble().toInteger()
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("build_area", val.toString())
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(dist._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
println row[17] +" "+val.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updadteAllAreaFieldByCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积(匹配).csv")
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
if(row[12] && (row[12]=="名称匹配"||row[12]=="映射别名匹配" || (row[12].contains("接口") && row[16] && row[16].toString().isNumber() && row[16].toString().toDouble()>=0.9))){
|
||||
def dist = CommonDatautils.findSeaweedById(row[13], ["all_area"])
|
||||
if(dist && !dist.all_area){
|
||||
def val = row[3]
|
||||
|
||||
if(!val){
|
||||
val = row[4]
|
||||
}
|
||||
if(!val){
|
||||
val = row[5]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[8]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[9]
|
||||
}
|
||||
if(!val){
|
||||
val = row[10]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[11]
|
||||
}
|
||||
if(!val){
|
||||
val = row[6]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[7]
|
||||
}
|
||||
if(val){
|
||||
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
|
||||
val = val.toString().toDouble().toInteger()
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("all_area", val.toString())
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(dist._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
println row[13] +" "+val.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updatePlotRateFieldByCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv")
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
if(row[22] && (row[22]=="名称匹配"||row[22]=="映射别名匹配" || (row[22].contains("接口") && row[26] && row[26].toString().isNumber() && row[26].toString().toDouble()>=0.9))){
|
||||
def dist = CommonDatautils.findSeaweedById(row[23], ["cubage_percent"])
|
||||
if(dist && !dist.cubage_percent){
|
||||
def val = row[10]
|
||||
|
||||
if(!val){
|
||||
val = row[11]
|
||||
}
|
||||
if(!val){
|
||||
val = row[12]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[19]
|
||||
}
|
||||
if(!val){
|
||||
val = row[20]
|
||||
}
|
||||
if(!val){
|
||||
val = row[21]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[6]
|
||||
}
|
||||
if(!val){
|
||||
val = row[8]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[9]
|
||||
}
|
||||
if(!val){
|
||||
val = row[13]
|
||||
}
|
||||
if(!val){
|
||||
val = row[14]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[17]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[3]
|
||||
}
|
||||
if(!val){
|
||||
val = row[4]
|
||||
}
|
||||
if(!val){
|
||||
val = row[5]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[15]
|
||||
}
|
||||
if(!val){
|
||||
val = row[16]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[18]
|
||||
}
|
||||
|
||||
if(val){
|
||||
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
|
||||
val = val.toString().toDouble().toInteger()
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("cubage_percent", val.toString())
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(dist._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
println row[23] +" "+val.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// def run1(){
|
||||
// def regionPointsMap = findRegionRange("重庆")
|
||||
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
// }
|
||||
|
||||
def updateFinishDateFieldByCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\竣工时间.csv")
|
||||
for(def i=0;i<datas.size();i++) {
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
if(row[22] && (row[22]=="名称匹配"||row[22]=="映射别名匹配" || (row[22].contains("接口") && row[26] && row[26].toString().isNumber() && row[26].toString().toDouble()>=0.9))){
|
||||
def dist = CommonDatautils.findSeaweedById(row[23], ["finish_building_date"])
|
||||
if(dist && !dist.finish_building_date){
|
||||
def val = row[10]
|
||||
|
||||
if(!val){
|
||||
val = row[11]
|
||||
}
|
||||
if(!val){
|
||||
val = row[12]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[19]
|
||||
}
|
||||
if(!val){
|
||||
val = row[20]
|
||||
}
|
||||
if(!val){
|
||||
val = row[21]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[6]
|
||||
}
|
||||
if(!val){
|
||||
val = row[8]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[9]
|
||||
}
|
||||
if(!val){
|
||||
val = row[13]
|
||||
}
|
||||
if(!val){
|
||||
val = row[14]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[17]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[3]
|
||||
}
|
||||
if(!val){
|
||||
val = row[4]
|
||||
}
|
||||
if(!val){
|
||||
val = row[5]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[15]
|
||||
}
|
||||
if(!val){
|
||||
val = row[16]
|
||||
}
|
||||
|
||||
if(!val){
|
||||
val = row[18]
|
||||
}
|
||||
|
||||
if(val){
|
||||
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
|
||||
val = val.toString().toDouble().toInteger()
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("finish_building_date", val.toString())
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(dist._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
println row[23] +" "+val.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def runCsv(){
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配.csv")//第三方网站绿化率 (3).csv
|
||||
def results = batchMatch(datas)
|
||||
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配-2.csv",null,"GBK")
|
||||
}
|
||||
|
||||
|
||||
def batchMatch(datas){
|
||||
def results = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
def row = data as List
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if (row[0] && row[1] && !row[22]) {
|
||||
def city = row[0]
|
||||
def region = row[1]
|
||||
def name = row[2]
|
||||
region = RegionData.formatRegion(city, region)
|
||||
if(region){
|
||||
if (name && name.contains("·")) {
|
||||
name = name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(city, region, name)
|
||||
|
||||
if (!dist) {
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name, "district")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if (!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
if (rets && rets.size() == 1) {
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
if (dist) {
|
||||
row << flag
|
||||
row << dist._id.toString()
|
||||
row << dist.region
|
||||
row << dist.name
|
||||
|
||||
results << row
|
||||
} else {
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null)
|
||||
if (jsonObject != null && jsonObject.precise != -1) {
|
||||
row << "匹配接口"
|
||||
row << jsonObject.get("id")
|
||||
row << jsonObject.get("district")
|
||||
row << jsonObject.get("name")
|
||||
row << jsonObject.get("credit")
|
||||
results << row
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println totalCount.get()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
|
||||
def filePath = "D:\\数据\\小区\\押品发现新增小区确认.xlsx"
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
|
||||
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx"
|
||||
def index = 0
|
||||
|
||||
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||
index++
|
||||
if(row['A'] && row['B']){
|
||||
def city = row['A']
|
||||
def name = row['B']
|
||||
def region = null
|
||||
def name = row['C']
|
||||
def region = row['B']
|
||||
|
||||
if(name && name.contains("·")){
|
||||
name = name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(city, region, name)
|
||||
@@ -203,26 +778,26 @@ class AnjukeDistrictMatch {
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = "匹配接口"
|
||||
row['G'] = jsonObject.get("id")
|
||||
row['D'] = jsonObject.get("district")
|
||||
row['C'] = jsonObject.get("name")
|
||||
row['Y'] = "匹配接口"
|
||||
row['Z'] = jsonObject.get("id")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
|
||||
if(row['G']){
|
||||
def dbo = CommonDatautils.findDistrictById(row['G'])
|
||||
if(dbo){
|
||||
row['H'] = dbo.estate_type2
|
||||
row['I'] = dbo.name_cat
|
||||
|
||||
def count = findBigdataCommunity(row['G'])
|
||||
if(count){
|
||||
row['E'] = "是"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println index +" "+row['Y']
|
||||
// if(row['G']){
|
||||
// def dbo = CommonDatautils.findDistrictById(row['G'])
|
||||
// if(dbo){
|
||||
// row['H'] = dbo.estate_type2
|
||||
// row['I'] = dbo.name_cat
|
||||
//
|
||||
// def count = findBigdataCommunity(row['G'])
|
||||
// if(count){
|
||||
// row['E'] = "是"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -286,6 +861,57 @@ class AnjukeDistrictMatch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def repeatDistByAlias(){
|
||||
def rets = findAliasDist()
|
||||
// def datas = []
|
||||
// for(def r : rets){
|
||||
// datas << [r.city, r.region, r.name, r.alias]
|
||||
// }
|
||||
// ExcelUtils.write("D:\\数据\\小区\\第三方有别名的小区.xlsx", datas, "test")
|
||||
def districts = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region","name"])
|
||||
def distKeys = []
|
||||
for(def d : districts){
|
||||
distKeys << d.region+"#"+d.name
|
||||
}
|
||||
|
||||
def results = []
|
||||
for(def t : rets){
|
||||
def names = []
|
||||
names << t.region + "#" +t.name
|
||||
def arrs = t.alias.split(",|,")
|
||||
for(def a : arrs){
|
||||
names << t.region + "#" + a
|
||||
if(a.contains("(")){
|
||||
names << t.region + "#" + a.replace("(","").replace(")", "")
|
||||
}
|
||||
}
|
||||
|
||||
def set = new HashSet()
|
||||
for(def n : names){
|
||||
if(distKeys.contains(n)){
|
||||
set << n
|
||||
}
|
||||
}
|
||||
|
||||
if(set.size()>1){
|
||||
results << [t.city, t.region, t.name, set.join(" "),t.url]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\小区\\第三方别名验证重复小区.xlsx", results, "test")
|
||||
}
|
||||
|
||||
def findAliasDist(){
|
||||
def rets = []
|
||||
def sql = "select * from community where city='常州'"
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
if(d.alias){
|
||||
rets << ["city":d.city, "region":d.region, "name":d.name, "source":d.source, "alias":d.alias, "url":d.url]
|
||||
}
|
||||
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAliasDatas(city, name){
|
||||
def must = []
|
||||
def term1 = ["match":["city":city]]
|
||||
@@ -309,4 +935,49 @@ class AnjukeDistrictMatch {
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split(",",-1)
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
def batchCreateFile(ret, fileName, headers, charset){
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
def sb = new StringBuffer();
|
||||
// sb.append(headers.join("@|@"))
|
||||
// sb.append(System.getProperty("line.separator"));
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row, ","))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"开始写入CSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"结束写入CSV "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
package com.fangjia.script.gj.poc
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
class AddPriceTest {
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new AddPriceTest();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\招商\\招商.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['M']){
|
||||
def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['M'], cityId)
|
||||
if(map){
|
||||
d['AK'] = map.avg_price
|
||||
d['AL'] = map.calc_type
|
||||
d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['AN'] = thirdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
if(!price && db.lianjia_esf_price && db.lianjia_esf_price>2000){
|
||||
price = db.lianjia_esf_price
|
||||
break;
|
||||
}
|
||||
|
||||
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
price = db.beike_esf_price
|
||||
break
|
||||
}
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
d['AO'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findGXDPrice(def city, region, name){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("source", "GXD")
|
||||
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
|
||||
def price = null
|
||||
for(def d : dbcursor){
|
||||
price = d.price
|
||||
}
|
||||
return price
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.script.gj.poc
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class RentPrice {
|
||||
|
||||
static main(args) {
|
||||
def task = new RentPrice();
|
||||
task.run("D:\\数据\\招商\\楼盘POC测试样例地址.xlsx");
|
||||
}
|
||||
def run(filepath){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
index++
|
||||
if(d['M']){
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if(dist){
|
||||
d['AZ'] = findHouses(dist.city, dist.region, "rent", 20221201,20230330, dist.name, dist.alias)
|
||||
}
|
||||
println index + " "+ d['AZ']
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
// inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
|
||||
// inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
|
||||
// queryBuilder.must(inBuilder)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def prices = []
|
||||
println city + " "+name+" "+houses.size()
|
||||
for(def d : houses){
|
||||
prices << d.avg_price
|
||||
}
|
||||
|
||||
if(prices && prices.size()>=3){
|
||||
return (int)(prices.sum()/prices.size())
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+2
-1
@@ -32,8 +32,9 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class CZDataBatch extends BaseBatch{
|
||||
|
||||
|
||||
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
||||
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
|
||||
|
||||
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
|
||||
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
|
||||
|
||||
+7
-1
@@ -31,7 +31,13 @@ class FzDataUtils {
|
||||
map.cat = "district";
|
||||
// 楼盘名称
|
||||
map.name = dist.name
|
||||
map.alias = dist.alias
|
||||
def alias = dist.alias
|
||||
if(alias){
|
||||
if(alias.size()>5){
|
||||
alias = alias[0..4]
|
||||
}
|
||||
map.alias = alias
|
||||
}
|
||||
map.build_date = getBuildDate(dist);
|
||||
//附属属性
|
||||
map.build_type = dist.build_type
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user