This commit is contained in:
2023-03-22 11:09:41 +08:00
21 changed files with 486 additions and 102 deletions
@@ -120,7 +120,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -56,7 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) // def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets) rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets) rows.addAll(rets)
@@ -76,7 +77,9 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle" println "txt 生成->导Oracle"
// saveToPgSql(rows)
// delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
} }
@@ -228,6 +231,11 @@ class BigdataCommunityBatch extends BaseBatch{
return ret; return ret;
} }
def delPgData(){
sqlInstance.execute("delete from base.t0831_bigdata_community_gj_h")
println "base.t0831_bigdata_community_gj_h 数据已删除"
}
def saveToPgSql(rows){ def saveToPgSql(rows){
for(def r : rows) { for(def r : rows) {
for (def i = 0; i < r.size(); i++) { for (def i = 0; i < r.size(); i++) {
@@ -236,7 +244,7 @@ class BigdataCommunityBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -694,7 +694,7 @@ class BigdataDealBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -58,7 +58,7 @@ class BigdataForsaleBatch extends BaseBatch{
} }
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
@@ -78,7 +78,7 @@ class BigdataForsaleBatch extends BaseBatch{
//目前查询库中是否已存在,用的是Oracle //目前查询库中是否已存在,用的是Oracle
def index = 0 def index = 0
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
citys = ["迪庆州"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -153,8 +153,8 @@ class BigdataForsaleBatch extends BaseBatch{
for(def i=0;i<districts.size();i++){ for(def i=0;i<districts.size();i++){
final def rMap = districts[i]; final def rMap = districts[i];
// fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
// void run() { void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def commId = rMap.get("_id").toString() def commId = rMap.get("_id").toString()
@@ -210,8 +210,8 @@ class BigdataForsaleBatch extends BaseBatch{
} finally { } finally {
latch.countDown() latch.countDown()
} }
// }; };
// }) })
} }
latch.await() latch.await()
return retMap return retMap
@@ -1106,11 +1106,11 @@ class BigdataForsaleBatch extends BaseBatch{
// 创建布隆过滤器对象 // 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet() // def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h where urbn_id='533400'",{it-> pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h",{it->
filter.put(it.gujia_sell_id) filter.put(it.gujia_sell_id)
}); });
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021 where urbn_id='533400'",{it-> pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021",{it->
filter.put(it.gujia_sell_id) filter.put(it.gujia_sell_id)
}); });
return filter; return filter;
@@ -558,6 +558,49 @@ class CcbCommonUtils {
} }
public static def findPgCcbDistrict(cityid){
def sql = "select * from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}'";
def map = [:]
pgsqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.gujiabdsocpoldtotarea
data << r.gujiabds_bld_tot_area
data << r.gujia_bds_grnrate
data << r.gujia_bds_vol_rate
data << r.gujia_bds_park_num
data << r.gujia_bds_bld_yr_rc
data << r.gujia_bds_hs_use_val
data << r.totunits
data << r.gujia_bds_data_tm_val
data << r.gujiabds_udt_flag_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_bds_st
map.put(r.gujia_comm_id, data)
});
return map
}
public static def findTradesByMonth(city, startDate, endDate){ public static def findTradesByMonth(city, startDate, endDate){
@@ -1,11 +1,14 @@
package com.fangjia.script.ccbevaluation package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql import groovy.sql.Sql
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
@@ -23,6 +26,9 @@ class FjwPricePkCcbPrice {
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) { static main(args) {
def task = new FjwPricePkCcbPrice() def task = new FjwPricePkCcbPrice()
// task.run() // task.run()
@@ -30,8 +36,48 @@ class FjwPricePkCcbPrice {
// task.addFjwPrice() // task.addFjwPrice()
// task.addCcbPrice() // task.addCcbPrice()
task.addCcbPriceFromGuided() // task.addCcbPriceFromGuided()
// task.addMarketPrice() // task.addMarketPrice()
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
// task.addCcbPrice2()
// task.addCcbPrice3()
task.standRegion()
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
if(d['R']){
def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
if(map){
d['AG'] = map.avg_price
d['AH'] = map.calc_type
d['AI'] = map.credit_value
d['AJ'] = map.month
}
}
})
}
def addCcbPrice3(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\瑞丰\\金华、绍兴国信达查价_最新建行价格.xlsx", { d ->
if(d['A']){
def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dist){
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(d['A']))
if(map){
d['F'] = map.avg_price
d['G'] = map.calc_type
d['H'] = map.month
}
}
}
}, 1)
} }
def addMarketPrice(){ def addMarketPrice(){
@@ -45,6 +91,62 @@ class FjwPricePkCcbPrice {
}) })
} }
def addNewDistPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新楼盘添加指导价.xlsx",{d->
// def map = findCommunityByEs(d['B'], d['C'], d['D'])
// if(map){
// if(map.get("安居客-新房")){
// def dbo = map.get("安居客-新房")
// d['F'] = dbo.avg_price
// d['G'] = dbo.open_date
// }
// if(map.get("贝壳找房-新房")){
// def dbo = map.get("贝壳找房-新房")
// d['H'] = dbo.avg_price
// d['I'] = dbo.open_date
// }else if(map.get("楼盘网-新房")){
// def dbo = map.get("楼盘网-新房")
// d['J'] = dbo.avg_price
// d['K'] = dbo.open_date
// }
// }
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
// def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
if(dbo){
d['N'] = dbo.avg_price
if(dbo.value_type == 1){
d['O'] = "指导价"
}else if(dbo.value_type == 2){
d['O'] = "自身样本"
}else if(dbo.value_type == 3){
d['O'] = "第三方"
}else if(dbo.value_type == 4){
d['O'] = "交易数据"
}else if(dbo.value_type == 5){
d['O'] = "周边楼盘"
}else if(dbo.value_type == 6){
d['O'] = "推导"
}else if(dbo.value_type == 9){
d['O'] = "神经网络"
}else if(dbo.value_type == 10){
d['O'] = "无方式"
}else{
d['O'] = ""
}
}
})
}
def findCommunityByEs(city,region, name){
def names = new HashSet()
def map = [:]
ElasticUtils.executeSqlQuery("select * from community where city='${city}' and region='${region}' and name='${name}' and is_new=1", { d->
map.put(d.source, d)
})
return map;
}
def addCcbPrice(){ def addCcbPrice(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d->
@@ -114,6 +216,15 @@ class FjwPricePkCcbPrice {
}) })
} }
def standRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入3月20日.xlsx",{d->
if(!d['J']){
def newRegion = RegionData.formatRegion(d['A'], d['B'])
d['J'] = newRegion
}
})
}
def run(){ def run(){
def ccbPrices = getCommunityBasePriceGj(202208) def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size() println "ccb总量:"+ccbPrices.size()
@@ -284,4 +395,78 @@ class FjwPricePkCcbPrice {
} }
out.close(); out.close();
} }
def exportFjwPriceAndCcbPrice(){
def citys = ["杭州"]
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name"]);
println c +" "+districts.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-05','yyyy-MM-dd'))
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def rets = []
rets << ["城市","区域","名称","房价网价格","房价网计算方式","建行价格","建行计算方式","建行计算时间"]
for(def dist : districts ){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if(priceMap){
ret << priceMap.avg_price
if(priceMap.value_type == 1){
ret << "指导价"
}else if(priceMap.value_type == 2){
ret << "自身样本"
}else if(priceMap.value_type == 3){
ret << "第三方"
}else if(priceMap.value_type == 4){
ret << "交易数据"
}else if(priceMap.value_type == 5){
ret << "周边楼盘"
}else if(priceMap.value_type == 6){
ret << "推导"
}else if(priceMap.value_type == 9){
ret << "神经网络"
}else if(priceMap.value_type == 10){
ret << "无方式"
}else{
ret << ""
}
}else{
ret << ''
ret << ''
}
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(c))
if(map){
ret << map.avg_price
ret << map.calc_type
ret << map.credit_value
ret << map.month
}
rets << ret
}
ExcelUtils.write("d:\\数据\\瑞丰\\杭州价格.xlsx",rets, c)
println c+" 导出完成"
}
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
} }
@@ -43,49 +43,49 @@ class GuiDingPriceBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args) // def citys = CcbCommonUtils.getRunningCitys(args)
def rows = [] // def rows = []
//
CcbCommonUtils.deleteGuidedPrice(curDate) // CcbCommonUtils.deleteGuidedPrice(curDate)
for (def city : citys) { // for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//
//添加自己库中的人工指导价 // //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-27") // def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") //// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ // if(prices){
def distMap = getDistrictMap(city) // def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) // def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets) // rows.addAll(rets)
println city + " " + rets.size() // println city + " " + rets.size()
}
//添加搜房的参考价
// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
// def datas = findFangDatas(districts)
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
// rows.addAll(fangRets)
//添加保亭文昌的指导价
// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
// def cityDataMap = [:]
// for(def d : cityDatas){
// cityDataMap.put(d['A'], d['E'])
// } // }
// def personDatas = findPersonDatas(districts, cityDataMap); //
// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) // //添加搜房的参考价
// rows.addAll(personRets) //// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
//// def datas = findFangDatas(districts)
} //// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
//// rows.addAll(fangRets)
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] //
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); // //添加保亭文昌的指导价
//// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
saveToPgSql(rows) //// def cityDataMap = [:]
//// for(def d : cityDatas){
//// cityDataMap.put(d['A'], d['E'])
//// }
//// def personDatas = findPersonDatas(districts, cityDataMap);
//// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
//// rows.addAll(personRets)
//
// }
//
// def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
//
// saveToPgSql(rows)
//更新静态数据表 //更新静态数据表
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
} }
@@ -224,7 +224,7 @@ class GuiDingPriceBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -245,7 +245,7 @@ class GuiDingPriceBatch extends BaseBatch{
} }
def updateVersion(def time){ def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h"] def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){ for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'" def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
pgsqlInstance.executeUpdate(sql) pgsqlInstance.executeUpdate(sql)
@@ -102,7 +102,7 @@ class StagingDistrictBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -71,6 +71,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
// //
//}) //})
def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
def allCitys = CcbCommonUtils.citys_all
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def c : allCitys){
def cityCode = cityCodeMap.get(c)
def count = null;
index++
for(def r : countCitys){
if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
count = r['B']
break
}
}
println index +" "+ c+" "+count
}
def findCcbPriceCredit(sqlInstance, cityid, commid){ def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:] def map = [:]
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch { class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def sDate= 202302//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202211 def startDate = 202212
def endDate = 202301 def endDate = 202302
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -632,7 +632,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
@@ -29,6 +30,9 @@ class BigdataCitypriceBatch extends BaseBatch{
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) { static main(args) {
runBatch(new BigdataCitypriceBatch(), args) runBatch(new BigdataCitypriceBatch(), args)
} }
@@ -64,7 +68,38 @@ class BigdataCitypriceBatch extends BaseBatch{
} }
def headers = [] def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_CITYPRICE 数据文件输出完成......" println "txt 生成-> BIGDATA_CITYPRICE"
// saveToPgSqlCityPrice(rets)
}
def saveToPgSqlCityPrice(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
ps.toString()
}
}
println "end: "+new Date()
} }
@@ -109,7 +144,37 @@ class BigdataCitypriceBatch extends BaseBatch{
} }
def headers = [] def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_DISTPRICE 数据文件输出完成......" println "txt 生成-> BIGDATA_DISTPRICE"
// saveToPgSqlRegionPrice(rets)
}
def saveToPgSqlRegionPrice(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}
}
println "end: "+new Date()
} }
def districtRun(citys, startDate, endDate, cityCodeDataMap){ def districtRun(citys, startDate, endDate, cityCodeDataMap){
@@ -140,9 +205,13 @@ class BigdataCitypriceBatch extends BaseBatch{
} }
def headers = [] def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_COMMPRICE 数据文件输出完成......" println "txt 生成->BIGDATA_COMMPRICE"
// saveToPgSqlDistrictPrice(rets)
} }
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
def distCount = districts.size() def distCount = districts.size()
@@ -172,6 +241,35 @@ class BigdataCitypriceBatch extends BaseBatch{
return rets; return rets;
} }
def saveToPgSqlDistrictPrice(rets){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}
}
println "end: "+new Date()
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def sellDataType = 1 def sellDataType = 1
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
} }
def calcMonthlyPrice(){ def calcMonthlyPrice(){
def citys = CommonUtils.findAllCity() def citys = ["福州"]//CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
index++ index++
def districts = [] def districts = []
if(city == "福州"){ if(city == "福州"){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList()
}else{ }else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
} }
@@ -117,8 +117,8 @@ class DistrictSellStatBatch extends BaseBatch {
def rets = new CopyOnWriteArrayList() def rets = new CopyOnWriteArrayList()
districts.each{dist-> districts.each{dist->
fixedThreadPool.execute(new Runnable() { // fixedThreadPool.execute(new Runnable() {
void run() { // void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name def key = dist.region +"##" + dist.name
@@ -202,8 +202,8 @@ class DistrictSellStatBatch extends BaseBatch {
} finally { } finally {
latch.countDown() latch.countDown()
} }
}; // };
}) // })
} }
latch.await() latch.await()
} }
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
task.priceMarkupRun(); task.priceMarkupRun();
task.regionPriceTrendRun() // task.regionPriceTrendRun()
//// task.margeExcel() //// task.margeExcel()
// task.saveCsvData() // task.saveCsvData()
} }
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -28,8 +28,8 @@ class CityStatExport {
def task = new CityStatExport(); def task = new CityStatExport();
// task.run() // task.run()
// task.run1() // task.run1()
// task.regionStat() task.regionStat()
task.stat() // task.stat()
} }
def stat(){ def stat(){
@@ -47,26 +47,26 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity(); // def citys = CommonDatautils.findAllCity();
def citys = ["太原","大同","临汾","朔州","忻州","晋中","阳泉","运城","长治","吕梁","晋城"] def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def ret = [] def ret = []
// ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"] ret << ["城市","小区总量"]
ret << ["城市","楼栋总量", "房号总量"] // ret << ["城市","楼栋总量", "房号总量"]
for(def city : citys){ for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
def total = districts.size() def total = districts.size()
def m = [] def m = []
m << city m << city
// m << total m << total
// def tradeCount = findTrades(city, '2022-02-01'); // def tradeCount = findTrades(city, '2022-02-01');
// m << tradeCount // m << tradeCount
// def houseCount = findHouseCount(city, '2022-02-01') // def houseCount = findHouseCount(city, '2022-02-01')
// m << houseCount // m << houseCount
def loudongCount = findLoudongCount(city) // def loudongCount = findLoudongCount(city)
m << loudongCount // m << loudongCount
//
def fanghaoCount = findFanghaoCount(city) // def fanghaoCount = findFanghaoCount(city)
m << fanghaoCount // m << fanghaoCount
// def priceCount = findDistrictPrice(city, null); // def priceCount = findDistrictPrice(city, null);
@@ -78,7 +78,7 @@ class CityStatExport {
ret << m ret << m
println m println m
} }
ExcelUtils.write("D:\\数据\\山西省城市数据统计(楼栋房号).xlsx",ret,"数据") ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
} }
@@ -96,12 +96,17 @@ class CityStatExport {
} }
def regionStat(){ def regionStat(){
def city = "无锡" def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def regions = CommonDatautils.getRegionListByCity(city) def ret = []
for(def r : regions){ ret << ["城市","区域","小区总量"]
def districts = CommonDatautils.findDistricts(city, r, null) for(def city : citys){
println r +" "+ districts.size() def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def districts = CommonDatautils.findDistricts(city, r, null)
println city +" "+ r +" "+ districts.size()
}
} }
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
} }
@@ -0,0 +1,28 @@
package com.fangjia.script.gj.district
import ch.qos.logback.core.db.dialect.DBUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
def opt = d['J']
if(opt && opt.contains("别名")){
def dist = CommonDatautils.findDistrictById(d['H'])
if(dist){
def alias = []
if(dist.alias){
alias = dist.alias
}
if(!alias.contains(d['C'])){
alias << d['C']
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
}
}
})
@@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles){ for(def e : cityEles){
def href = e.attr("href") def href = e.attr("href")
if(e.text().contains("ÖØÇì")){ // if(e.text().contains("¸£ÖÝ")){
def cityMap = [:] def cityMap = [:]
cityMap.city = e.text() cityMap.city = e.text()
cityMap.url = "https:"+href cityMap.url = "https:"+href
list << cityMap list << cityMap
// cityMap.put(e.text(), "https:"+href) // cityMap.put(e.text(), "https:"+href)
} // }
} }
} }
return list return list
@@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource()); def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:] def cityMap = [:]
def citys= ["ÖØÇì"] def citys= ["¸£ÖÝ"]
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){ if (!href.contains("fang.lianjia") ){
cityMap.put(e.text(), href) cityMap.put(e.text(), href)
} }
} }
@@ -40,11 +40,11 @@ class DataStat {
def run(){ def run(){
def source = "365ÌÔ·¿"// def source = "ËÑ·¿"//
def citys = findCrawlCity(source, 202211); def citys = findCrawlCity(source, 202301);
for(def city : citys){ for(def city : citys){
def count1 = findCount(source, city, 202211); def count1 = findCount(source, city, 202301);
def count2 = findCount(source, city, 202212); def count2 = findCount(source, city, 202302);
println city +" "+count1+" "+count2 println city +" "+count1+" "+count2
// println count1 // println count1
// println "--- ----" // println "--- ----"
@@ -26,7 +26,8 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] // def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["搜房":"搜房"]
static main(args) { static main(args) {
@@ -42,8 +43,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......" println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity() def citys = ["凉山州"]//CommonUtils.findAllCity()
esSources.each{esSource, mgSource-> esSources.each{esSource, mgSource->
for(def city : citys){ for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource, mdate) def rets = findAllEsData(city, esSource, mgSource, mdate)
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){ def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){ for(def source :sources){
def count1 = findTotalCount(source, 202212); def count1 = findTotalCount(source, 202301);
def count2 = findTotalCount(source, 202301); def count2 = findTotalCount(source, 202302);
println source +" "+count1 +" "+count2 println source +" "+count1 +" "+count2
} }
} }