This commit is contained in:
2023-04-10 11:28:39 +08:00
15 changed files with 1610 additions and 86 deletions
@@ -440,7 +440,7 @@ class CcbCommonUtils {
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
println "原始:"+dbsour.count()
println "[${city}] 原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
@@ -450,7 +450,7 @@ class CcbCommonUtils {
rets << d
}
}
println "小区量:"+rets.size()
println "[${city}] 小区量:"+rets.size()
return rets;
}
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
// task.addFjwPrice()
task.addFjwPrice()
// task.addCcbPrice()
// task.addCcbPriceFromGuided()
@@ -44,7 +44,7 @@ class FjwPricePkCcbPrice {
// task.addCcbPrice2()
// task.addCcbPrice3()
task.standRegion()
// task.standRegion()
}
def addCcbPrice2(){
@@ -202,13 +202,13 @@ class FjwPricePkCcbPrice {
}
def addFjwPrice(){
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
if(!d['M']){
def dist = CommonDatautils.findDistrictById(d['D'])
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
if(dbo){
d['M'] = dbo.avg_price
d['K'] = dbo.avg_price
}
}
}
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
@@ -28,25 +29,41 @@ class BigdataCitypriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def curRunDate = null
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def cityCodeMap = CcbCommonUtils.findCityCodeValue()
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
}
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
cityRun(citys, startDate, endDate, cityCodeDataMap)
regionRun(citys, startDate, endDate, cityCodeDataMap)
districtRun(citys, startDate, endDate, cityCodeDataMap)
def curDate = new Date()
def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args)
if(args){
def dateStr = args[0]
curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01
}
println "args 参数:"+args.toString() +" 城市数量:"+citys.size()+" curDate: "+ curDate
curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger();
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
def cityRun(citys, startDate, endDate, cityCodeDataMap){
@@ -177,11 +194,18 @@ class BigdataCitypriceBatch extends BaseBatch{
println "end: "+new Date()
}
def districtRun(citys, startDate, endDate, cityCodeDataMap){
def districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap){
def rets = []
def allDistricts = []
def allPriceMap = [:]
def allRentMap = [:]
def allDealCountMap = [:]
def allCityRankMap = [:]
def allRegionRankMap = [:]
def allDistCityPreMap = [:]
def allDistRegionPreMap = [:]
def allLastMonthMap = [:]
def allLastYearMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
allDistricts.addAll(districts)
@@ -192,9 +216,36 @@ class BigdataCitypriceBatch extends BaseBatch{
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
allRentMap = allRentMap + rentMap
def cityCode = cityCodeMap.get(city)
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
def cityRankMap = findCommunityCityRank(datas)
allCityRankMap = allCityRankMap + cityRankMap
def regionRankMap = findCommunityRegionRank(datas)
allRegionRankMap = allRegionRankMap + regionRankMap
def cityPrice = findCcbCityPrice(cityCode, startDate.toString().substring(0,6))
def regionPriceMap = findCcbRegionPrice(cityCode, startDate.toString().substring(0,6))
def cityPreMap = districtCityPrictRate(cityPrice, datas)
allDistCityPreMap = allDistCityPreMap + cityPreMap
def regionPreMap = districtRegionPriceRate(regionPriceMap, datas)
allDistRegionPreMap = allDistRegionPreMap + regionPreMap
def lastMonthMap = findDistQuotations(cityCode, dealStartDate.replace("-","").substring(0,6))
allLastMonthMap = allLastMonthMap + lastMonthMap
def lastYearMonthMap = findDistQuotations(cityCode, lastYearDate.toString().substring(0,6))
allLastYearMap = allLastYearMap + lastYearMonthMap
}
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap)
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
if(cityLists){
rets.addAll(cityLists)
}
@@ -212,7 +263,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
final CountDownLatch latch = new CountDownLatch(distCount)
@@ -220,11 +271,12 @@ class BigdataCitypriceBatch extends BaseBatch{
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
if(row){
rets << row
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
@@ -271,7 +323,7 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def sellDataType = 1
def rentDataType = 1
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
@@ -400,6 +452,160 @@ class BigdataCitypriceBatch extends BaseBatch{
row << 1
row << 'FJW'
//成交样本量,成交样本量同比, 成交样本量环比
if(dealCountMap){
def dealCount = dealCountMap.get(dist._id.toString())
row << dealCount?dealCount:''
def dealMarkup = ''
def dealYoy = ''
if(allLastMonthMap){
def dealMarkupMap = allLastMonthMap.get(dist._id.toString())
try{
if(dealCount && dealMarkupMap && dealMarkupMap.deal_count && dealMarkupMap.deal_count>0){
def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count
dealMarkup = DataHandleUtils.getDoubleRound(2, markup)
}
}catch(Exception e){
e.printStackTrace()
}
}
if(allLastYearMap){
def dealYoyMap = allLastYearMap.get(dist._id.toString())
if(dealCount && dealYoyMap && dealYoyMap.deal_count && dealYoyMap.deal_count>0){
def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count
dealYoy = DataHandleUtils.getDoubleRound(2, markup)
}
}
row << dealYoy
row << dealMarkup
}else{
row << ''
row << ''
row << ''
}
//出售平均单价的同比,环比
if(sell_avg_price){
def markup = ''
if(allLastMonthMap){
def dataMap = allLastMonthMap.get(dist._id.toString())
if(dataMap && dataMap.saleprice){
markup = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice
markup = DataHandleUtils.getDoubleRound(2, markup)
}
}
def yoy = ''
if(allLastYearMap){
def dataMap = allLastYearMap.get(dist._id.toString())
if(dataMap && dataMap.saleprice){
yoy = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice
yoy = DataHandleUtils.getDoubleRound(2, yoy)
}
}
row << yoy
row << markup
}else{
row << ''
row << ''
}
//出售样本量同比,环比
if(sell_count){
def markup = ''
if(allLastMonthMap){
def dataMap = allLastMonthMap.get(dist._id.toString())
if(dataMap && dataMap.salecount){
markup = (sell_count - dataMap.salecount)*100/dataMap.salecount
markup = DataHandleUtils.getDoubleRound(2, markup)
}
}
def yoy = ''
if(allLastYearMap){
def dataMap = allLastYearMap.get(dist._id.toString())
if(dataMap && dataMap.salecount){
yoy = (sell_count - dataMap.salecount)*100/dataMap.salecount
yoy = DataHandleUtils.getDoubleRound(2, yoy)
}
}
row << yoy
row << markup
}else{
row << ''
row << ''
}
//出租房源量同比,环比
if(rent_count){
def markup = ''
if(allLastMonthMap){
def dataMap = allLastMonthMap.get(dist._id.toString())
if(dataMap && dataMap.salecount && dataMap.rentcount>0){
markup = (rent_count - dataMap.rentcount)*100/dataMap.rentcount
markup = DataHandleUtils.getDoubleRound(2, markup)
}
}
def yoy = ''
if(allLastYearMap){
def dataMap = allLastYearMap.get(dist._id.toString())
if(dataMap && dataMap.salecount && dataMap.rentcount>0){
yoy = (rent_count - dataMap.rentcount)*100/dataMap.rentcount
yoy = DataHandleUtils.getDoubleRound(2, yoy)
}
}
row << yoy
row << markup
}else{
row << ''
row << ''
}
//小区价格城市排名
if(allCityRankMap){
def cityRank = allCityRankMap.get(dist._id.toString())
if(cityRank){
row << cityRank
}else{
row << ''
}
}else{
row << ''
}
//小区价格区域排名
if(allRegionRankMap){
def regionRank = allRegionRankMap.get(dist._id.toString())
if(regionRank){
row << regionRank
}else{
row << ''
}
}else{
row << ''
}
//高于区县平均价比率
if(allDistRegionPreMap){
def regionRadio = allDistRegionPreMap.get(dist._id.toString())
if(regionRadio){
row << regionRadio
}else{
row << ''
}
}else{
row << ''
}
//高于城市平均价比率
if(allDistCityPreMap){
def cityRadio = allDistCityPreMap.get(dist._id.toString())
if(cityRadio){
row << cityRadio
}else{
row << ''
}
}else{
row << ''
}
return row
}
return null
@@ -723,6 +929,135 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def findDealDatesByCity(cityCode, startDate, endDate){
def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id";
def rMap = [:]
pgsqlInstance.eachRow(sql, { r ->
rMap.put(r.gujia_comm_id, r.count)
})
return rMap;
}
def findCommunityCityRank(datas){
def values = []
for(def m : datas){
values << m.avg_price
}
values = values.sort().reverse()
def resMap = [:]
def total = values.size()
for(def m : datas){
// def index = 0
// for(def v : values){
// index++
// if(m.avg_price == v){
// break;
// }
// }
def index = values.indexOf(m.avg_price)+1
resMap.put(m.comm_id, index+"|"+total)
}
return resMap
}
def findCommunityRegionRank(datas){
def regionMap = [:]
for(def m : datas){
def prices = regionMap.get(m.region)
if(!prices){
prices = []
regionMap.put(m.region, prices)
}
prices << m.avg_price
prices.sort().reverse()
}
def resMap = [:]
for(def m : datas){
def prices = regionMap.get(m.region)
def total = prices.size()
def index = prices.indexOf(m.avg_price)+1
// for(def v : prices){
// index++
// if(m.avg_price == v){
// break;
// }
// }
resMap.put(m.comm_id, index+"|"+total)
}
return resMap
}
def findCommunityPriceByCity(cityCode, month){
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def datas = []
pgsqlInstance.eachRow(sql, { r ->
datas << ["comm_id":r.gujia_bds_id, "avg_price":r.gujia_bds_clc_avprc, "region":r.gujia_admndiv_nm]
})
return datas;
}
def findCcbCityPrice(cityCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def avgPrice = null
pgsqlInstance.eachRow(sql, { r ->
avgPrice = r.gujia_bds_clc_avprc
})
return avgPrice;
}
def findCcbRegionPrice(cityCode, month){
def sql = "SELECT gxda_admndiv_nm,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def regionPriceMap = [:]
pgsqlInstance.eachRow(sql, { r ->
regionPriceMap.put(r.gxda_admndiv_nm, r.gujia_bds_clc_avprc)
})
return regionPriceMap;
}
def districtCityPrictRate(cityPrice, datas){
def distCityRateMap = [:]
if(cityPrice){
for(def m : datas){
def val = (m.avg_price-cityPrice)*100/cityPrice
distCityRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val))
}
}
return distCityRateMap;
}
def districtRegionPriceRate(regionPriceMap, datas){
def distRegionRateMap = [:]
if(regionPriceMap){
for(def m : datas){
def regionAvgPrice = regionPriceMap.get(m.region)
if(regionAvgPrice){
def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice
distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val))
}else{
println m.region+" 无区域均价"
}
}
}
return distRegionRateMap;
}
def findDistQuotations(cityCode, month){
def sql = "select commid, deal_count, saleprice,salecount,rentcount from bigdata_commprice_fjw where ymid=${month} and cityid=${cityCode}";
def markupMap = [:]
oracleSqlInstance.eachRow(sql, { r ->
markupMap.put(r.commid, ["deal_count":r.DEAL_COUNT, "saleprice":r.SALEPRICE,"salecount":r.SALECOUNT,"rentcount":r.RENTCOUNT])
})
return markupMap;
}
@@ -85,7 +85,7 @@ class ExportDistrictRongHe extends BaseBatch{
exportPgBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportPgBigdataForsale("2023-01-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
SSHHelper jschCommand = new SSHHelper();
@@ -1669,7 +1669,7 @@ class ExportDistrictRongHe extends BaseBatch{
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE,DEAL_COUNT,DEAL_COUNT_YOY,DEAL_COUNT_MOM,NETLIST_PRICE_YOY,NETLIST_PRICE_MOM,NETLIST_COUNT_YOY,NETLIST_COUNT_MOM,RENT_COUNT_YOY,RENT_COUNT_MOM,CITY_RANK,AREA_RANK,DISTRICT_PRICE_THAN,CITY_PRICE_THAN from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -1697,6 +1697,19 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.RDATERANGE
data << r.DATATYPE
data << r.SOURCECODE
data << r.DEAL_COUNT
data << r.DEAL_COUNT_YOY
data << r.DEAL_COUNT_MOM
data << r.NETLIST_PRICE_YOY
data << r.NETLIST_PRICE_MOM
data << r.NETLIST_COUNT_YOY
data << r.NETLIST_COUNT_MOM
data << r.RENT_COUNT_YOY
data << r.RENT_COUNT_MOM
data << r.CITY_RANK
data << r.AREA_RANK
data << r.DISTRICT_PRICE_THAN
data << r.CITY_PRICE_THAN
allRows << data
});
}
@@ -5,6 +5,7 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@@ -14,6 +15,9 @@ import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHeTest extends BaseBatch{
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
@@ -26,36 +30,37 @@ class ExportDistrictRongHeTest extends BaseBatch{
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// if(curDate.toString().endsWith("10")){
exportEvaluateRate()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
// exportEvaluateRate()
// exportBigdataBuildprojFjw()
// exportBigdataCommlayoutpicFjw()
//
//
// exportBigdataCommunityGJRH()/
// exportBigdataBuildingFJW()//
// exportBigdataCommfeeGj()//
//
// exportBigdataCommbuildstruGJ()//
// exportBigdataCommhousetypeGJ()//
// exportBigdataCommbuildyearGj()//
// exportBigdataCommhouseuseGJ()//
// exportBigdataPositionGJ()//
// exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()//
//
//
// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
exportInOk()
@@ -528,6 +533,50 @@ class ExportDistrictRongHeTest extends BaseBatch{
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def exportPgBigdataDealFJW(curDate, dealdate, new_dists){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealdate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";// and txn_dt>='${dealdate}'
allRows.addAll(dealData(sql))
}
}
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def createDeal(sql){
def allRows = []
sqlInstance.eachRow(sql,{r->
@@ -607,6 +656,87 @@ class ExportDistrictRongHeTest extends BaseBatch{
return allRows;
}
def dealData(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_bds_mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_bds_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujiabdstot_floor_num
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_bds_trdtckt_prc
data << r.gujia_bds_mdl_tot_prc
data << r.gujia_bds_dcl_tot_prc
data << r.gujiabds_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_bds_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiabdslndfgrngvarea
data << r.gujiabdslutbl_gv_area
data << r.gujiabdssuntblgv_area
data << r.gujiabdsbayindowgarea
data << r.gujiabdstowerfrgvarea
data << r.gujia_bds_grg_gv_area
data << r.gujiabdsundgdrmgvarea
data << r.gujiabds_othr_gv_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
// data << r.gujia_bds_mkdam
data << r.gujia_bds_data_tm_val
allRows << data
});
return allRows
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
@@ -760,7 +890,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
@@ -807,7 +937,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate}
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}'";// and YMID=${queryDate}
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -854,7 +984,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate}
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID>=202210";// and YMID=${queryDate}
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -0,0 +1,74 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
class UpdateCommPrice {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def cityCodeMap = CcbCommonUtils.findCityCodeValue()
static main(args) {
def task = new UpdateCommPrice()
task.run(args)
}
def run(Object args){
def citys = CcbCommonUtils.citys_all
def curDate = new Date()
if(args){
def dateStr = args[0]
curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01
}
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger();
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
process(citys, dealStartDate, startDate, curRunDate)
}
def process(citys, dealStartDate, startDate, curRunDate){
def rets = []
def allDistricts = []
def allDealCountMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias"]);
allDistricts.addAll(districts)
def cityCode = cityCodeMap.get(city)
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
}
for(def i=0;i<allDistricts.size();i++) {
final def dist = allDistricts[i];
def dealCount = allDealCountMap.get(dist._id.toString())
if(dealCount){
updateDealCount(dist._id.toString(), dealCount, curRunDate)
}
}
}
def updateDealCount(def dist_id, count, month){
def sql = "update bigdata_commprice_fjw set deal_count=${count} where ymid=${month} and commid='${dist_id}'"
oracleSqlInstance.execute(sql)
}
def findDealDatesByCity(cityCode, startDate, endDate){
def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id";
def rMap = [:]
pgsqlInstance.eachRow(sql, { r ->
rMap.put(r.gujia_comm_id, r.count)
})
return rMap;
}
}
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
}
def calcMonthlyPrice(){
def citys = ["福州"]//CommonUtils.findAllCity()
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
index++
def districts = []
if(city == "福州"){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList()
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
}
@@ -18,13 +18,13 @@ class AddAvgpriceBatch {
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static {
citys = CommonDatautils.findAllCity()
citys = ["徐州"]//CommonDatautils.findAllCity()
println "加载城市完成:"+citys.size()
}
static main(args) {
def task = new AddAvgpriceBatch();
task.run();
task.run1();
}
def run(){
@@ -63,6 +63,45 @@ class AddAvgpriceBatch {
def run1(){
def totalCount = 0;
citys.each{city->
if(city!="全国"){
def districtKeysMap = CommonDatautils.findAllDistrictKeys(city, "district")
println "------------------------------------------------------------"
println "${city}小区量:"+districtKeysMap.size()
if(districtKeysMap){
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26"))
println "【2023-03-26】共计均价小区:"+lastWeekendStats.size()
//库中小区没有计算过均价的小区
def districtKeys = districtKeysMap.keySet();
def rets1 = findNoCaluteDistrict(districtKeys, lastWeekendStats);
println "【2023-03-26】共 "+rets1.size()+" 个小区无均价"
if(rets1){
for(def disKey : rets1){
def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd'))
}
}
}
}
}
println "共补:"+totalCount
}
def addPrice(city, region, name, estateType2,startDate, endDate){
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
}
}
def findHavePrice(lastAgoWeekendStats, rets){
def disKeys = [];
for(def key : rets){
@@ -0,0 +1,40 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
def rMap = [:]
def index = 0
OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
index++
def day = doc.c_date.format("yyyy-MM-dd")
def count = rMap.get(day)
if (!count) {
rMap.put(day, 1)
} else {
rMap.put(day, count + 1)
}
if(index%10000==0){
println index+" "+new Date().toString()
}
}
rMap.each {k,val->
println k+" "+val
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.linxsh.util.AddressUtil
@@ -11,26 +12,71 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AnjukeDistrictMatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new AnjukeDistrictMatch();
// task.run();
task.matchCommunity()
// task.runCsv();
// task.updateCredit()
// task.matchCommunity()//查询es小区数据
// task.run1()
// task.updateField()
// task.updateGreenRateField()//根据第三方数据填补库中字段
// task.updteGreenRateFieldByCsv()
// task.fenQiRun()
// task.findAliasDatas("淮安","水沐未来印象花园")
// task.findAliasDatas("淮安","水沐未来印象花园")//模糊查找Es别名数据
// task.repeatDistByAlias()//根据第三方网站别名查找重复数据
// task.matchCommunityByCsv()
// task.updteBuildAreaFieldByCsv()
// task.updadteAllAreaFieldByCsv()
// task.updatePlotRateFieldByCsv()
task.updateFinishDateFieldByCsv()
}
def updateCredit(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv")
def results = []
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[24] && row[24].contains("接口")){
def inputName = row[2]
def matchName = row[27]
if(inputName.toString().endsWith("小区") || inputName.toString().endsWith("院")){
inputName = inputName.replace("小区","").replace("院","")
}
if(matchName.toString().endsWith("小区") || matchName.toString().endsWith("院")){
matchName = matchName.replace("小区","").replace("院","")
}
if(inputName == matchName && row[1]==row[26]){
row[30] = 1
}
}
results << row
}
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv",null,"GBK")
batchCreateFile(datas, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-2.csv",null,"GBK")
}
@@ -96,7 +142,7 @@ class AnjukeDistrictMatch {
}
def updateField(){
def updateGreenRateField(){
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['Q'] && !row['Q'].contains("接口")){
@@ -155,18 +201,547 @@ class AnjukeDistrictMatch {
})
}
def matchCommunityByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv")
def map = [:]
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
map.put(row[0]+"#"+row[1]+"#"+row[2], row[24]+"#"+row[25]+"#"+row[26]+"#"+row[27]+"#"+row[28])
}
def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间.csv")
def lastRets = []
for(def i=0;i<datas1.size();i++) {
final def data = datas1[i]
def row = data as List
def resultStr = map.get(row[0]+"#"+row[1]+"#"+row[2])
if(resultStr){
def arrs = resultStr.split("#")
if(arrs){
row << arrs[0]
if(arrs.size()>1){
row << arrs[1]
}
if(arrs.size()>2){
row << arrs[2]
}
if(arrs.size()>3){
row << arrs[3]
}
if(arrs.size()>4){
row << arrs[4]
}
}
}
lastRets << row
}
batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间(匹配).csv",null,"GBK")
}
def updteGreenRateFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\测试.csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[24] && (row[24]=="名称匹配"||row[24]=="映射别名匹配" || (row[24].contains("接口") && row[28] && row[28].toString().isNumber() && row[28].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[25], ["virescence_percent"])
if(dist && !dist.virescence_percent){
def val = row[12]
if(!val){
val = row[13]
}
if(!val){
val = row[14]
}
if(!val){
val = row[11]
}
if(!val){
val = row[21]
}
if(!val){
val = row[22]
}
if(!val){
val = row[6]
}
if(!val){
val = row[10]
}
if(!val){
val = row[9]
}
if(!val){
val = row[5]
}
if(!val){
val = row[16]
}
if(!val){
val = row[15]
}
if(!val){
val = row[23]
}
if(!val){
val = row[19]
}
if(!val){
val = row[17]
}
if(!val){
val = row[8]
}
if(!val){
val = row[18]
}
if(!val){
val = row[20]
}
if(!val){
val = row[3]
}
if(!val){
val = row[4]
}
if(!val){
val = row[7]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("virescence_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[25] +" "+val.toString()
}
}
}
}
}
def updteBuildAreaFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站建筑面积(匹配).csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[16] && (row[16]=="名称匹配"||row[16]=="映射别名匹配" || (row[16].contains("接口") && row[20] && row[20].toString().isNumber() && row[20].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[17], ["build_area"])
if(dist && !dist.build_area){
def val = row[8]
if(!val){
val = row[7]
}
if(!val){
val = row[9]
}
if(!val){
val = row[14]
}
if(!val){
val = row[15]
}
if(!val){
val = row[12]
}
if(!val){
val = row[5]
}
if(!val){
val = row[6]
}
if(!val){
val = row[3]
}
if(!val){
val = row[13]
}
if(!val){
val = row[4]
}
if(!val){
val = row[10]
}
if(!val){
val = row[11]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("build_area", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[17] +" "+val.toString()
}
}
}
}
}
def updadteAllAreaFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积(匹配).csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[12] && (row[12]=="名称匹配"||row[12]=="映射别名匹配" || (row[12].contains("接口") && row[16] && row[16].toString().isNumber() && row[16].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[13], ["all_area"])
if(dist && !dist.all_area){
def val = row[3]
if(!val){
val = row[4]
}
if(!val){
val = row[5]
}
if(!val){
val = row[8]
}
if(!val){
val = row[9]
}
if(!val){
val = row[10]
}
if(!val){
val = row[11]
}
if(!val){
val = row[6]
}
if(!val){
val = row[7]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("all_area", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[13] +" "+val.toString()
}
}
}
}
}
def updatePlotRateFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[22] && (row[22]=="名称匹配"||row[22]=="映射别名匹配" || (row[22].contains("接口") && row[26] && row[26].toString().isNumber() && row[26].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[23], ["cubage_percent"])
if(dist && !dist.cubage_percent){
def val = row[10]
if(!val){
val = row[11]
}
if(!val){
val = row[12]
}
if(!val){
val = row[19]
}
if(!val){
val = row[20]
}
if(!val){
val = row[21]
}
if(!val){
val = row[6]
}
if(!val){
val = row[8]
}
if(!val){
val = row[9]
}
if(!val){
val = row[13]
}
if(!val){
val = row[14]
}
if(!val){
val = row[17]
}
if(!val){
val = row[3]
}
if(!val){
val = row[4]
}
if(!val){
val = row[5]
}
if(!val){
val = row[15]
}
if(!val){
val = row[16]
}
if(!val){
val = row[18]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("cubage_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[23] +" "+val.toString()
}
}
}
}
}
// def run1(){
// def regionPointsMap = findRegionRange("重庆")
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
// }
def updateFinishDateFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\竣工时间.csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[22] && (row[22]=="名称匹配"||row[22]=="映射别名匹配" || (row[22].contains("接口") && row[26] && row[26].toString().isNumber() && row[26].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[23], ["finish_building_date"])
if(dist && !dist.finish_building_date){
def val = row[10]
if(!val){
val = row[11]
}
if(!val){
val = row[12]
}
if(!val){
val = row[19]
}
if(!val){
val = row[20]
}
if(!val){
val = row[21]
}
if(!val){
val = row[6]
}
if(!val){
val = row[8]
}
if(!val){
val = row[9]
}
if(!val){
val = row[13]
}
if(!val){
val = row[14]
}
if(!val){
val = row[17]
}
if(!val){
val = row[3]
}
if(!val){
val = row[4]
}
if(!val){
val = row[5]
}
if(!val){
val = row[15]
}
if(!val){
val = row[16]
}
if(!val){
val = row[18]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("finish_building_date", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[23] +" "+val.toString()
}
}
}
}
}
def runCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配.csv")//第三方网站绿化率 (3).csv
def results = batchMatch(datas)
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配-2.csv",null,"GBK")
}
def batchMatch(datas){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def data = datas[i]
def row = data as List
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if (row[0] && row[1] && !row[22]) {
def city = row[0]
def region = row[1]
def name = row[2]
region = RegionData.formatRegion(city, region)
if(region){
if (name && name.contains("·")) {
name = name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(city, region, name)
if (!dist) {
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name, "district")
flag = "映射别名匹配"
}
if (!dist) {
def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if (rets && rets.size() == 1) {
dist = rets[0]
}
}
if (dist) {
row << flag
row << dist._id.toString()
row << dist.region
row << dist.name
results << row
} else {
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null)
if (jsonObject != null && jsonObject.precise != -1) {
row << "匹配接口"
row << jsonObject.get("id")
row << jsonObject.get("district")
row << jsonObject.get("name")
row << jsonObject.get("credit")
results << row
}
}
}
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return results;
}
def run(){
def filePath = "D:\\数据\\小区\\押品发现新增小区确认.xlsx"
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(filePath,{row->
index++
if(row['A'] && row['B']){
def city = row['A']
def name = row['B']
def region = null
def name = row['C']
def region = row['B']
if(name && name.contains("·")){
name = name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(city, region, name)
@@ -203,26 +778,26 @@ class AnjukeDistrictMatch {
}else{
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = "匹配接口"
row['G'] = jsonObject.get("id")
row['D'] = jsonObject.get("district")
row['C'] = jsonObject.get("name")
row['Y'] = "匹配接口"
row['Z'] = jsonObject.get("id")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("credit")
}
}
if(row['G']){
def dbo = CommonDatautils.findDistrictById(row['G'])
if(dbo){
row['H'] = dbo.estate_type2
row['I'] = dbo.name_cat
def count = findBigdataCommunity(row['G'])
if(count){
row['E'] = "是"
}
}
}
println index +" "+row['Y']
// if(row['G']){
// def dbo = CommonDatautils.findDistrictById(row['G'])
// if(dbo){
// row['H'] = dbo.estate_type2
// row['I'] = dbo.name_cat
//
// def count = findBigdataCommunity(row['G'])
// if(count){
// row['E'] = "是"
// }
// }
// }
}
})
}
@@ -286,6 +861,57 @@ class AnjukeDistrictMatch {
return rets;
}
def repeatDistByAlias(){
def rets = findAliasDist()
// def datas = []
// for(def r : rets){
// datas << [r.city, r.region, r.name, r.alias]
// }
// ExcelUtils.write("D:\\数据\\小区\\第三方有别名的小区.xlsx", datas, "test")
def districts = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region","name"])
def distKeys = []
for(def d : districts){
distKeys << d.region+"#"+d.name
}
def results = []
for(def t : rets){
def names = []
names << t.region + "#" +t.name
def arrs = t.alias.split(",|,")
for(def a : arrs){
names << t.region + "#" + a
if(a.contains("(")){
names << t.region + "#" + a.replace("(","").replace(")", "")
}
}
def set = new HashSet()
for(def n : names){
if(distKeys.contains(n)){
set << n
}
}
if(set.size()>1){
results << [t.city, t.region, t.name, set.join(" "),t.url]
}
}
ExcelUtils.write("D:\\数据\\小区\\第三方别名验证重复小区.xlsx", results, "test")
}
def findAliasDist(){
def rets = []
def sql = "select * from community where city='常州'"
ElasticUtils.executeSqlQuery(sql, { d->
if(d.alias){
rets << ["city":d.city, "region":d.region, "name":d.name, "source":d.source, "alias":d.alias, "url":d.url]
}
})
return rets;
}
def findAliasDatas(city, name){
def must = []
def term1 = ["match":["city":city]]
@@ -309,4 +935,49 @@ class AnjukeDistrictMatch {
}
return rets;
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split(",",-1)
rows << row
}
return rows;
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row, ","))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -0,0 +1,131 @@
package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class AddPriceTest {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
static main(args) {
def task = new AddPriceTest();
task.run();
}
def run(){
def filePath = "D:\\数据\\招商\\招商.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addThirdPrice(filePath)
}
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['M']){
def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['M'], cityId)
if(map){
d['AK'] = map.avg_price
d['AL'] = map.calc_type
d['AM'] = map.credit_value
}
}
})
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['AN'] = thirdPrice
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
if(!price && db.lianjia_esf_price && db.lianjia_esf_price>2000){
price = db.lianjia_esf_price
break;
}
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
price = db.beike_esf_price
break
}
}
return price
}
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
d['AO'] = gxdPrice
}
}
})
}
def findGXDPrice(def city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
for(def d : dbcursor){
price = d.price
}
return price
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.gj.poc
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class RentPrice {
static main(args) {
def task = new RentPrice();
task.run("D:\\数据\\招商\\楼盘POC测试样例地址.xlsx");
}
def run(filepath){
def index = 0
ExcelUtils.multiThreadingModify(filepath, { d ->
index++
if(d['M']){
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
d['AZ'] = findHouses(dist.city, dist.region, "rent", 20221201,20230330, dist.name, dist.alias)
}
println index + " "+ d['AZ']
}
})
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
// inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
// inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
// queryBuilder.must(inBuilder)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def prices = []
println city + " "+name+" "+houses.size()
for(def d : houses){
prices << d.avg_price
}
if(prices && prices.size()>=3){
return (int)(prices.sum()/prices.size())
}
return null
}
}
@@ -32,8 +32,9 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class CZDataBatch extends BaseBatch{
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
@@ -31,7 +31,13 @@ class FzDataUtils {
map.cat = "district";
// 楼盘名称
map.name = dist.name
map.alias = dist.alias
def alias = dist.alias
if(alias){
if(alias.size()>5){
alias = alias[0..4]
}
map.alias = alias
}
map.build_date = getBuildDate(dist);
//附属属性
map.build_type = dist.build_type
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")