Merge branch 'script_gj' into 'master'

建行导小区修改

See merge request !572
This commit is contained in:
2023-03-20 17:10:23 +08:00
20 changed files with 434 additions and 50 deletions
@@ -120,7 +120,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -56,7 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) // def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets) rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets) rows.addAll(rets)
@@ -76,7 +77,9 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle" println "txt 生成->导Oracle"
// saveToPgSql(rows)
// delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
} }
@@ -228,6 +231,11 @@ class BigdataCommunityBatch extends BaseBatch{
return ret; return ret;
} }
def delPgData(){
sqlInstance.execute("delete from base.t0831_bigdata_community_gj_h")
println "base.t0831_bigdata_community_gj_h 数据已删除"
}
def saveToPgSql(rows){ def saveToPgSql(rows){
for(def r : rows) { for(def r : rows) {
for (def i = 0; i < r.size(); i++) { for (def i = 0; i < r.size(); i++) {
@@ -236,7 +244,7 @@ class BigdataCommunityBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -247,7 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{
println "start: "+new Date() println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(20000, sql){ ps-> sqlInstance.withBatch(10, sql){ ps->
for(def r : rows) { for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
@@ -694,7 +694,7 @@ class BigdataDealBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -558,6 +558,49 @@ class CcbCommonUtils {
} }
public static def findPgCcbDistrict(cityid){
def sql = "select * from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}'";
def map = [:]
pgsqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.gujiabdsocpoldtotarea
data << r.gujiabds_bld_tot_area
data << r.gujia_bds_grnrate
data << r.gujia_bds_vol_rate
data << r.gujia_bds_park_num
data << r.gujia_bds_bld_yr_rc
data << r.gujia_bds_hs_use_val
data << r.totunits
data << r.gujia_bds_data_tm_val
data << r.gujiabds_udt_flag_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_bds_st
map.put(r.gujia_comm_id, data)
});
return map
}
public static def findTradesByMonth(city, startDate, endDate){ public static def findTradesByMonth(city, startDate, endDate){
@@ -1,11 +1,14 @@
package com.fangjia.script.ccbevaluation package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql import groovy.sql.Sql
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
@@ -23,6 +26,9 @@ class FjwPricePkCcbPrice {
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) { static main(args) {
def task = new FjwPricePkCcbPrice() def task = new FjwPricePkCcbPrice()
// task.run() // task.run()
@@ -30,8 +36,48 @@ class FjwPricePkCcbPrice {
// task.addFjwPrice() // task.addFjwPrice()
// task.addCcbPrice() // task.addCcbPrice()
task.addCcbPriceFromGuided() // task.addCcbPriceFromGuided()
// task.addMarketPrice() // task.addMarketPrice()
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
// task.addCcbPrice2()
// task.addCcbPrice3()
task.standRegion()
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
if(d['R']){
def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
if(map){
d['AG'] = map.avg_price
d['AH'] = map.calc_type
d['AI'] = map.credit_value
d['AJ'] = map.month
}
}
})
}
def addCcbPrice3(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\瑞丰\\金华、绍兴国信达查价_最新建行价格.xlsx", { d ->
if(d['A']){
def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dist){
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(d['A']))
if(map){
d['F'] = map.avg_price
d['G'] = map.calc_type
d['H'] = map.month
}
}
}
}, 1)
} }
def addMarketPrice(){ def addMarketPrice(){
@@ -45,6 +91,62 @@ class FjwPricePkCcbPrice {
}) })
} }
def addNewDistPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新楼盘添加指导价.xlsx",{d->
// def map = findCommunityByEs(d['B'], d['C'], d['D'])
// if(map){
// if(map.get("安居客-新房")){
// def dbo = map.get("安居客-新房")
// d['F'] = dbo.avg_price
// d['G'] = dbo.open_date
// }
// if(map.get("贝壳找房-新房")){
// def dbo = map.get("贝壳找房-新房")
// d['H'] = dbo.avg_price
// d['I'] = dbo.open_date
// }else if(map.get("楼盘网-新房")){
// def dbo = map.get("楼盘网-新房")
// d['J'] = dbo.avg_price
// d['K'] = dbo.open_date
// }
// }
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
// def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
if(dbo){
d['N'] = dbo.avg_price
if(dbo.value_type == 1){
d['O'] = "指导价"
}else if(dbo.value_type == 2){
d['O'] = "自身样本"
}else if(dbo.value_type == 3){
d['O'] = "第三方"
}else if(dbo.value_type == 4){
d['O'] = "交易数据"
}else if(dbo.value_type == 5){
d['O'] = "周边楼盘"
}else if(dbo.value_type == 6){
d['O'] = "推导"
}else if(dbo.value_type == 9){
d['O'] = "神经网络"
}else if(dbo.value_type == 10){
d['O'] = "无方式"
}else{
d['O'] = ""
}
}
})
}
def findCommunityByEs(city,region, name){
def names = new HashSet()
def map = [:]
ElasticUtils.executeSqlQuery("select * from community where city='${city}' and region='${region}' and name='${name}' and is_new=1", { d->
map.put(d.source, d)
})
return map;
}
def addCcbPrice(){ def addCcbPrice(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d->
@@ -114,6 +216,15 @@ class FjwPricePkCcbPrice {
}) })
} }
def standRegion(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\写字楼入clickhouse.xlsx",{d->
if(!d['S']){
def newRegion = RegionData.formatRegion(d['A'], d['B'].replace("区区","区"))
d['X'] = newRegion
}
})
}
def run(){ def run(){
def ccbPrices = getCommunityBasePriceGj(202208) def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size() println "ccb总量:"+ccbPrices.size()
@@ -284,4 +395,78 @@ class FjwPricePkCcbPrice {
} }
out.close(); out.close();
} }
def exportFjwPriceAndCcbPrice(){
def citys = ["杭州"]
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name"]);
println c +" "+districts.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-05','yyyy-MM-dd'))
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def rets = []
rets << ["城市","区域","名称","房价网价格","房价网计算方式","建行价格","建行计算方式","建行计算时间"]
for(def dist : districts ){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if(priceMap){
ret << priceMap.avg_price
if(priceMap.value_type == 1){
ret << "指导价"
}else if(priceMap.value_type == 2){
ret << "自身样本"
}else if(priceMap.value_type == 3){
ret << "第三方"
}else if(priceMap.value_type == 4){
ret << "交易数据"
}else if(priceMap.value_type == 5){
ret << "周边楼盘"
}else if(priceMap.value_type == 6){
ret << "推导"
}else if(priceMap.value_type == 9){
ret << "神经网络"
}else if(priceMap.value_type == 10){
ret << "无方式"
}else{
ret << ""
}
}else{
ret << ''
ret << ''
}
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(c))
if(map){
ret << map.avg_price
ret << map.calc_type
ret << map.credit_value
ret << map.month
}
rets << ret
}
ExcelUtils.write("d:\\数据\\瑞丰\\杭州价格.xlsx",rets, c)
println c+" 导出完成"
}
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
} }
@@ -224,7 +224,7 @@ class GuiDingPriceBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -102,7 +102,7 @@ class StagingDistrictBatch extends BaseBatch{
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -71,6 +71,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
// //
//}) //})
def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
def allCitys = CcbCommonUtils.citys_all
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def c : allCitys){
def cityCode = cityCodeMap.get(c)
def count = null;
index++
for(def r : countCitys){
if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
count = r['B']
break
}
}
println index +" "+ c+" "+count
}
def findCcbPriceCredit(sqlInstance, cityid, commid){ def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:] def map = [:]
@@ -632,7 +632,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
} }
} }
r << null r << null
r << '2023-02-28' r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null r << null
r << '2999-12-31' r << '2999-12-31'
r << null r << null
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
@@ -29,6 +30,9 @@ class BigdataCitypriceBatch extends BaseBatch{
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) { static main(args) {
runBatch(new BigdataCitypriceBatch(), args) runBatch(new BigdataCitypriceBatch(), args)
} }
@@ -64,7 +68,38 @@ class BigdataCitypriceBatch extends BaseBatch{
} }
def headers = [] def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_CITYPRICE 数据文件输出完成......" println "txt 生成-> BIGDATA_CITYPRICE"
// saveToPgSqlCityPrice(rets)
}
def saveToPgSqlCityPrice(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
ps.toString()
}
}
println "end: "+new Date()
} }
@@ -109,7 +144,37 @@ class BigdataCitypriceBatch extends BaseBatch{
} }
def headers = [] def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_DISTPRICE 数据文件输出完成......" println "txt 生成-> BIGDATA_DISTPRICE"
// saveToPgSqlRegionPrice(rets)
}
def saveToPgSqlRegionPrice(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}
}
println "end: "+new Date()
} }
def districtRun(citys, startDate, endDate, cityCodeDataMap){ def districtRun(citys, startDate, endDate, cityCodeDataMap){
@@ -140,9 +205,13 @@ class BigdataCitypriceBatch extends BaseBatch{
} }
def headers = [] def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_COMMPRICE 数据文件输出完成......" println "txt 生成->BIGDATA_COMMPRICE"
// saveToPgSqlDistrictPrice(rets)
} }
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
def distCount = districts.size() def distCount = districts.size()
@@ -172,6 +241,35 @@ class BigdataCitypriceBatch extends BaseBatch{
return rets; return rets;
} }
def saveToPgSqlDistrictPrice(rets){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}
}
println "end: "+new Date()
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def sellDataType = 1 def sellDataType = 1
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
} }
def calcMonthlyPrice(){ def calcMonthlyPrice(){
def citys = CommonUtils.findAllCity() def citys = ["福州"]//CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
index++ index++
def districts = [] def districts = []
if(city == "福州"){ if(city == "福州"){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList()
}else{ }else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
} }
@@ -117,8 +117,8 @@ class DistrictSellStatBatch extends BaseBatch {
def rets = new CopyOnWriteArrayList() def rets = new CopyOnWriteArrayList()
districts.each{dist-> districts.each{dist->
fixedThreadPool.execute(new Runnable() { // fixedThreadPool.execute(new Runnable() {
void run() { // void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name def key = dist.region +"##" + dist.name
@@ -202,8 +202,8 @@ class DistrictSellStatBatch extends BaseBatch {
} finally { } finally {
latch.countDown() latch.countDown()
} }
}; // };
}) // })
} }
latch.await() latch.await()
} }
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
task.priceMarkupRun(); task.priceMarkupRun();
task.regionPriceTrendRun() // task.regionPriceTrendRun()
//// task.margeExcel() //// task.margeExcel()
// task.saveCsvData() // task.saveCsvData()
} }
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -28,8 +28,8 @@ class CityStatExport {
def task = new CityStatExport(); def task = new CityStatExport();
// task.run() // task.run()
// task.run1() // task.run1()
// task.regionStat() task.regionStat()
task.stat() // task.stat()
} }
def stat(){ def stat(){
@@ -47,26 +47,26 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity(); // def citys = CommonDatautils.findAllCity();
def citys = ["太原","大同","临汾","朔州","忻州","晋中","阳泉","运城","长治","吕梁","晋城"] def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def ret = [] def ret = []
// ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"] ret << ["城市","小区总量"]
ret << ["城市","楼栋总量", "房号总量"] // ret << ["城市","楼栋总量", "房号总量"]
for(def city : citys){ for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
def total = districts.size() def total = districts.size()
def m = [] def m = []
m << city m << city
// m << total m << total
// def tradeCount = findTrades(city, '2022-02-01'); // def tradeCount = findTrades(city, '2022-02-01');
// m << tradeCount // m << tradeCount
// def houseCount = findHouseCount(city, '2022-02-01') // def houseCount = findHouseCount(city, '2022-02-01')
// m << houseCount // m << houseCount
def loudongCount = findLoudongCount(city) // def loudongCount = findLoudongCount(city)
m << loudongCount // m << loudongCount
//
def fanghaoCount = findFanghaoCount(city) // def fanghaoCount = findFanghaoCount(city)
m << fanghaoCount // m << fanghaoCount
// def priceCount = findDistrictPrice(city, null); // def priceCount = findDistrictPrice(city, null);
@@ -78,7 +78,7 @@ class CityStatExport {
ret << m ret << m
println m println m
} }
ExcelUtils.write("D:\\数据\\山西省城市数据统计(楼栋房号).xlsx",ret,"数据") ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
} }
@@ -96,12 +96,17 @@ class CityStatExport {
} }
def regionStat(){ def regionStat(){
def city = "无锡" def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def regions = CommonDatautils.getRegionListByCity(city) def ret = []
for(def r : regions){ ret << ["城市","区域","小区总量"]
def districts = CommonDatautils.findDistricts(city, r, null) for(def city : citys){
println r +" "+ districts.size() def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def districts = CommonDatautils.findDistricts(city, r, null)
println city +" "+ r +" "+ districts.size()
}
} }
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
} }
@@ -0,0 +1,28 @@
package com.fangjia.script.gj.district
import ch.qos.logback.core.db.dialect.DBUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
def opt = d['J']
if(opt && opt.contains("别名")){
def dist = CommonDatautils.findDistrictById(d['H'])
if(dist){
def alias = []
if(dist.alias){
alias = dist.alias
}
if(!alias.contains(d['C'])){
alias << d['C']
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
}
}
})
@@ -401,7 +401,7 @@ public class BeiKeCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles){ for(def e : cityEles){
def href = e.attr("href") def href = e.attr("href")
if(e.text().contains("重庆")){ if(e.text().contains("福州")){
def cityMap = [:] def cityMap = [:]
cityMap.city = e.text() cityMap.city = e.text()
cityMap.url = "https:"+href cityMap.url = "https:"+href
@@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource()); def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:] def cityMap = [:]
def citys= ["重庆"] def citys= ["福州"]
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){ if (!href.contains("fang.lianjia") ){
cityMap.put(e.text(), href) cityMap.put(e.text(), href)
} }
} }
@@ -40,11 +40,11 @@ class DataStat {
def run(){ def run(){
def source = "365淘房"// def source = "搜房"//
def citys = findCrawlCity(source, 202211); def citys = findCrawlCity(source, 202301);
for(def city : citys){ for(def city : citys){
def count1 = findCount(source, city, 202211); def count1 = findCount(source, city, 202301);
def count2 = findCount(source, city, 202212); def count2 = findCount(source, city, 202302);
println city +" "+count1+" "+count2 println city +" "+count1+" "+count2
// println count1 // println count1
// println "--- ----" // println "--- ----"
@@ -26,7 +26,8 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] // def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["搜房":"搜房"]
static main(args) { static main(args) {
@@ -42,8 +43,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......" println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity() def citys = ["凉山州"]//CommonUtils.findAllCity()
esSources.each{esSource, mgSource-> esSources.each{esSource, mgSource->
for(def city : citys){ for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource, mdate) def rets = findAllEsData(city, esSource, mgSource, mdate)
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){ def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){ for(def source :sources){
def count1 = findTotalCount(source, 202212); def count1 = findTotalCount(source, 202301);
def count2 = findTotalCount(source, 202301); def count2 = findTotalCount(source, 202302);
println source +" "+count1 +" "+count2 println source +" "+count1 +" "+count2
} }
} }