+1
-1
@@ -120,7 +120,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
|
||||
+12
-4
@@ -56,7 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
||||
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city))
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -76,7 +77,9 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
// saveToPgSql(rows)
|
||||
|
||||
// delPgData()
|
||||
saveToPgSql(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
}
|
||||
|
||||
@@ -228,6 +231,11 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
return ret;
|
||||
}
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.t0831_bigdata_community_gj_h")
|
||||
println "base.t0831_bigdata_community_gj_h 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
@@ -236,7 +244,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
@@ -247,7 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
sqlInstance.withBatch(10, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
|
||||
+1
-1
@@ -694,7 +694,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
|
||||
+43
@@ -558,6 +558,49 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
|
||||
public static def findPgCcbDistrict(cityid){
|
||||
def sql = "select * from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}'";
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.comm_nm
|
||||
data << r.gujia_dtl_adr
|
||||
data << r.gujia_bds_ltt
|
||||
data << r.gujia_bds_lgt
|
||||
data << r.gujiabdsocpoldtotarea
|
||||
data << r.gujiabds_bld_tot_area
|
||||
data << r.gujia_bds_grnrate
|
||||
data << r.gujia_bds_vol_rate
|
||||
data << r.gujia_bds_park_num
|
||||
data << r.gujia_bds_bld_yr_rc
|
||||
data << r.gujia_bds_hs_use_val
|
||||
data << r.totunits
|
||||
data << r.gujia_bds_data_tm_val
|
||||
data << r.gujiabds_udt_flag_val
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_comm_tp_val
|
||||
data << r.gujia_comm_fmnm
|
||||
data << r.gujia_str
|
||||
data << r.gujia_hsno
|
||||
data << r.gujiawthr_new_pan_ind
|
||||
data << r.gujia_comm_intd
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.gujia_src_co
|
||||
data << r.gujia_vlgtwn_str_mng
|
||||
data << r.gujia_lgc_admndiv_cd
|
||||
data << r.gujia_lgc_adiv
|
||||
data << r.gujia_bds_bds_st
|
||||
map.put(r.gujia_comm_id, data)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static def findTradesByMonth(city, startDate, endDate){
|
||||
|
||||
+186
-1
@@ -1,11 +1,14 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
@@ -23,6 +26,9 @@ class FjwPricePkCcbPrice {
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
@@ -30,8 +36,48 @@ class FjwPricePkCcbPrice {
|
||||
// task.addFjwPrice()
|
||||
// task.addCcbPrice()
|
||||
|
||||
task.addCcbPriceFromGuided()
|
||||
// task.addCcbPriceFromGuided()
|
||||
// task.addMarketPrice()
|
||||
|
||||
// task.exportFjwPriceAndCcbPrice()
|
||||
// task.addNewDistPrice()
|
||||
|
||||
// task.addCcbPrice2()
|
||||
// task.addCcbPrice3()
|
||||
task.standRegion()
|
||||
}
|
||||
|
||||
def addCcbPrice2(){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
|
||||
if(d['R']){
|
||||
def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
|
||||
if(map){
|
||||
d['AG'] = map.avg_price
|
||||
d['AH'] = map.calc_type
|
||||
d['AI'] = map.credit_value
|
||||
d['AJ'] = map.month
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addCcbPrice3(){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\瑞丰\\金华、绍兴国信达查价_最新建行价格.xlsx", { d ->
|
||||
if(d['A']){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(dist){
|
||||
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(d['A']))
|
||||
if(map){
|
||||
d['F'] = map.avg_price
|
||||
d['G'] = map.calc_type
|
||||
d['H'] = map.month
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}, 1)
|
||||
}
|
||||
|
||||
def addMarketPrice(){
|
||||
@@ -45,6 +91,62 @@ class FjwPricePkCcbPrice {
|
||||
})
|
||||
}
|
||||
|
||||
def addNewDistPrice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新楼盘添加指导价.xlsx",{d->
|
||||
// def map = findCommunityByEs(d['B'], d['C'], d['D'])
|
||||
// if(map){
|
||||
// if(map.get("安居客-新房")){
|
||||
// def dbo = map.get("安居客-新房")
|
||||
// d['F'] = dbo.avg_price
|
||||
// d['G'] = dbo.open_date
|
||||
// }
|
||||
// if(map.get("贝壳找房-新房")){
|
||||
// def dbo = map.get("贝壳找房-新房")
|
||||
// d['H'] = dbo.avg_price
|
||||
// d['I'] = dbo.open_date
|
||||
// }else if(map.get("楼盘网-新房")){
|
||||
// def dbo = map.get("楼盘网-新房")
|
||||
// d['J'] = dbo.avg_price
|
||||
// d['K'] = dbo.open_date
|
||||
// }
|
||||
// }
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
|
||||
// def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
|
||||
if(dbo){
|
||||
d['N'] = dbo.avg_price
|
||||
|
||||
if(dbo.value_type == 1){
|
||||
d['O'] = "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
d['O'] = "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
d['O'] = "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
d['O'] = "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
d['O'] = "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
d['O'] = "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
d['O'] = "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
d['O'] = "无方式"
|
||||
}else{
|
||||
d['O'] = ""
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findCommunityByEs(city,region, name){
|
||||
def names = new HashSet()
|
||||
def map = [:]
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${city}' and region='${region}' and name='${name}' and is_new=1", { d->
|
||||
map.put(d.source, d)
|
||||
})
|
||||
return map;
|
||||
}
|
||||
|
||||
def addCcbPrice(){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d->
|
||||
@@ -114,6 +216,15 @@ class FjwPricePkCcbPrice {
|
||||
})
|
||||
}
|
||||
|
||||
def standRegion(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\写字楼入clickhouse.xlsx",{d->
|
||||
if(!d['S']){
|
||||
def newRegion = RegionData.formatRegion(d['A'], d['B'].replace("区区","区"))
|
||||
d['X'] = newRegion
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def ccbPrices = getCommunityBasePriceGj(202208)
|
||||
println "ccb总量:"+ccbPrices.size()
|
||||
@@ -284,4 +395,78 @@ class FjwPricePkCcbPrice {
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
def exportFjwPriceAndCcbPrice(){
|
||||
def citys = ["杭州"]
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name"]);
|
||||
println c +" "+districts.size()
|
||||
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-05','yyyy-MM-dd'))
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","房价网价格","房价网计算方式","建行价格","建行计算方式","建行计算时间"]
|
||||
for(def dist : districts ){
|
||||
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
if(priceMap){
|
||||
ret << priceMap.avg_price
|
||||
if(priceMap.value_type == 1){
|
||||
ret << "指导价"
|
||||
}else if(priceMap.value_type == 2){
|
||||
ret << "自身样本"
|
||||
}else if(priceMap.value_type == 3){
|
||||
ret << "第三方"
|
||||
}else if(priceMap.value_type == 4){
|
||||
ret << "交易数据"
|
||||
}else if(priceMap.value_type == 5){
|
||||
ret << "周边楼盘"
|
||||
}else if(priceMap.value_type == 6){
|
||||
ret << "推导"
|
||||
}else if(priceMap.value_type == 9){
|
||||
ret << "神经网络"
|
||||
}else if(priceMap.value_type == 10){
|
||||
ret << "无方式"
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ''
|
||||
ret << ''
|
||||
}
|
||||
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(c))
|
||||
if(map){
|
||||
ret << map.avg_price
|
||||
ret << map.calc_type
|
||||
ret << map.credit_value
|
||||
ret << map.month
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\瑞丰\\杭州价格.xlsx",rets, c)
|
||||
println c+" 导出完成"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -224,7 +224,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
|
||||
@@ -71,6 +71,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
//
|
||||
//})
|
||||
|
||||
def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
|
||||
def allCitys = CcbCommonUtils.citys_all
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def c : allCitys){
|
||||
def cityCode = cityCodeMap.get(c)
|
||||
def count = null;
|
||||
index++
|
||||
for(def r : countCitys){
|
||||
if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
|
||||
count = r['B']
|
||||
break
|
||||
}
|
||||
}
|
||||
println index +" "+ c+" "+count
|
||||
}
|
||||
|
||||
def findCcbPriceCredit(sqlInstance, cityid, commid){
|
||||
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
|
||||
def map = [:]
|
||||
|
||||
+1
-1
@@ -632,7 +632,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
|
||||
+101
-3
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -29,6 +30,9 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCitypriceBatch(), args)
|
||||
}
|
||||
@@ -64,7 +68,38 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_CITYPRICE 数据文件输出完成......"
|
||||
println "txt 生成-> BIGDATA_CITYPRICE"
|
||||
|
||||
// saveToPgSqlCityPrice(rets)
|
||||
}
|
||||
|
||||
def saveToPgSqlCityPrice(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(1000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
@@ -109,7 +144,37 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_DISTPRICE 数据文件输出完成......"
|
||||
println "txt 生成-> BIGDATA_DISTPRICE"
|
||||
// saveToPgSqlRegionPrice(rets)
|
||||
}
|
||||
|
||||
def saveToPgSqlRegionPrice(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(1000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def districtRun(citys, startDate, endDate, cityCodeDataMap){
|
||||
@@ -140,9 +205,13 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_COMMPRICE 数据文件输出完成......"
|
||||
println "txt 生成->BIGDATA_COMMPRICE"
|
||||
|
||||
// saveToPgSqlDistrictPrice(rets)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
|
||||
def totalCount = new AtomicLong()
|
||||
def distCount = districts.size()
|
||||
@@ -172,6 +241,35 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def saveToPgSqlDistrictPrice(rets){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(1000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
|
||||
def sellDataType = 1
|
||||
|
||||
+6
-6
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def calcMonthlyPrice(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
|
||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
||||
@@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
index++
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList()
|
||||
}else{
|
||||
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
|
||||
}
|
||||
@@ -117,8 +117,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
districts.each{dist->
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def key = dist.region +"##" + dist.name
|
||||
@@ -202,8 +202,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
// };
|
||||
// })
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
task.regionPriceTrendRun()
|
||||
// task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+22
-17
@@ -28,8 +28,8 @@ class CityStatExport {
|
||||
def task = new CityStatExport();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.regionStat()
|
||||
task.stat()
|
||||
task.regionStat()
|
||||
// task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -47,26 +47,26 @@ class CityStatExport {
|
||||
// def citys = CommonDatautils.findAllCity();
|
||||
|
||||
|
||||
def citys = ["太原","大同","临汾","朔州","忻州","晋中","阳泉","运城","长治","吕梁","晋城"]
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def ret = []
|
||||
// ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"]
|
||||
ret << ["城市","楼栋总量", "房号总量"]
|
||||
ret << ["城市","小区总量"]
|
||||
// ret << ["城市","楼栋总量", "房号总量"]
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
def total = districts.size()
|
||||
def m = []
|
||||
m << city
|
||||
// m << total
|
||||
m << total
|
||||
// def tradeCount = findTrades(city, '2022-02-01');
|
||||
// m << tradeCount
|
||||
// def houseCount = findHouseCount(city, '2022-02-01')
|
||||
// m << houseCount
|
||||
|
||||
def loudongCount = findLoudongCount(city)
|
||||
m << loudongCount
|
||||
|
||||
def fanghaoCount = findFanghaoCount(city)
|
||||
m << fanghaoCount
|
||||
// def loudongCount = findLoudongCount(city)
|
||||
// m << loudongCount
|
||||
//
|
||||
// def fanghaoCount = findFanghaoCount(city)
|
||||
// m << fanghaoCount
|
||||
|
||||
|
||||
// def priceCount = findDistrictPrice(city, null);
|
||||
@@ -78,7 +78,7 @@ class CityStatExport {
|
||||
ret << m
|
||||
println m
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\山西省城市数据统计(楼栋房号).xlsx",ret,"数据")
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
|
||||
|
||||
}
|
||||
|
||||
@@ -96,12 +96,17 @@ class CityStatExport {
|
||||
}
|
||||
|
||||
def regionStat(){
|
||||
def city = "无锡"
|
||||
def regions = CommonDatautils.getRegionListByCity(city)
|
||||
for(def r : regions){
|
||||
def districts = CommonDatautils.findDistricts(city, r, null)
|
||||
println r +" "+ districts.size()
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def ret = []
|
||||
ret << ["城市","区域","小区总量"]
|
||||
for(def city : citys){
|
||||
def regions = CommonDatautils.getRegionListByCity(city)
|
||||
for(def r : regions){
|
||||
def districts = CommonDatautils.findDistricts(city, r, null)
|
||||
println city +" "+ r +" "+ districts.size()
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
|
||||
}
|
||||
|
||||
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import ch.qos.logback.core.db.dialect.DBUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
|
||||
def opt = d['J']
|
||||
if(opt && opt.contains("别名")){
|
||||
def dist = CommonDatautils.findDistrictById(d['H'])
|
||||
if(dist){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
alias = dist.alias
|
||||
}
|
||||
if(!alias.contains(d['C'])){
|
||||
alias << d['C']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
+1
-1
@@ -401,7 +401,7 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("重庆")){
|
||||
if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
|
||||
+2
-2
@@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["重庆"]
|
||||
def citys= ["福州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia") ){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "365淘房"//
|
||||
def citys = findCrawlCity(source, 202211);
|
||||
def source = "搜房"//
|
||||
def citys = findCrawlCity(source, 202301);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202211);
|
||||
def count2 = findCount(source, city, 202212);
|
||||
def count1 = findCount(source, city, 202301);
|
||||
def count2 = findCount(source, city, 202302);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+3
-3
@@ -26,7 +26,8 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
|
||||
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
|
||||
def esSources = ["搜房":"搜房"]
|
||||
|
||||
|
||||
static main(args) {
|
||||
@@ -42,8 +43,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
|
||||
println "starting......"
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def citys = CommonUtils.findAllCity()
|
||||
|
||||
def citys = ["凉山州"]//CommonUtils.findAllCity()
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
def rets = findAllEsData(city, esSource, mgSource, mdate)
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202212);
|
||||
def count2 = findTotalCount(source, 202301);
|
||||
def count1 = findTotalCount(source, 202301);
|
||||
def count2 = findTotalCount(source, 202302);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user