建行导小区修改

This commit is contained in:
2023-03-20 17:09:48 +08:00
parent ba3a73743a
commit 08ff0e34fe
20 changed files with 434 additions and 50 deletions
@@ -120,7 +120,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -56,7 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -76,7 +77,9 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
// saveToPgSql(rows)
// delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
@@ -228,6 +231,11 @@ class BigdataCommunityBatch extends BaseBatch{
return ret;
}
def delPgData(){
sqlInstance.execute("delete from base.t0831_bigdata_community_gj_h")
println "base.t0831_bigdata_community_gj_h 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
@@ -236,7 +244,7 @@ class BigdataCommunityBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -247,7 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(20000, sql){ ps->
sqlInstance.withBatch(10, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
@@ -694,7 +694,7 @@ class BigdataDealBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -558,6 +558,49 @@ class CcbCommonUtils {
}
public static def findPgCcbDistrict(cityid){
def sql = "select * from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}'";
def map = [:]
pgsqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.gujiabdsocpoldtotarea
data << r.gujiabds_bld_tot_area
data << r.gujia_bds_grnrate
data << r.gujia_bds_vol_rate
data << r.gujia_bds_park_num
data << r.gujia_bds_bld_yr_rc
data << r.gujia_bds_hs_use_val
data << r.totunits
data << r.gujia_bds_data_tm_val
data << r.gujiabds_udt_flag_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_bds_st
map.put(r.gujia_comm_id, data)
});
return map
}
public static def findTradesByMonth(city, startDate, endDate){
@@ -1,11 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
@@ -23,6 +26,9 @@ class FjwPricePkCcbPrice {
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new FjwPricePkCcbPrice()
// task.run()
@@ -30,8 +36,48 @@ class FjwPricePkCcbPrice {
// task.addFjwPrice()
// task.addCcbPrice()
task.addCcbPriceFromGuided()
// task.addCcbPriceFromGuided()
// task.addMarketPrice()
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
// task.addCcbPrice2()
// task.addCcbPrice3()
task.standRegion()
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
if(d['R']){
def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
if(map){
d['AG'] = map.avg_price
d['AH'] = map.calc_type
d['AI'] = map.credit_value
d['AJ'] = map.month
}
}
})
}
def addCcbPrice3(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\瑞丰\\金华、绍兴国信达查价_最新建行价格.xlsx", { d ->
if(d['A']){
def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dist){
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(d['A']))
if(map){
d['F'] = map.avg_price
d['G'] = map.calc_type
d['H'] = map.month
}
}
}
}, 1)
}
def addMarketPrice(){
@@ -45,6 +91,62 @@ class FjwPricePkCcbPrice {
})
}
def addNewDistPrice(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新楼盘添加指导价.xlsx",{d->
// def map = findCommunityByEs(d['B'], d['C'], d['D'])
// if(map){
// if(map.get("安居客-新房")){
// def dbo = map.get("安居客-新房")
// d['F'] = dbo.avg_price
// d['G'] = dbo.open_date
// }
// if(map.get("贝壳找房-新房")){
// def dbo = map.get("贝壳找房-新房")
// d['H'] = dbo.avg_price
// d['I'] = dbo.open_date
// }else if(map.get("楼盘网-新房")){
// def dbo = map.get("楼盘网-新房")
// d['J'] = dbo.avg_price
// d['K'] = dbo.open_date
// }
// }
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
// def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd'))
if(dbo){
d['N'] = dbo.avg_price
if(dbo.value_type == 1){
d['O'] = "指导价"
}else if(dbo.value_type == 2){
d['O'] = "自身样本"
}else if(dbo.value_type == 3){
d['O'] = "第三方"
}else if(dbo.value_type == 4){
d['O'] = "交易数据"
}else if(dbo.value_type == 5){
d['O'] = "周边楼盘"
}else if(dbo.value_type == 6){
d['O'] = "推导"
}else if(dbo.value_type == 9){
d['O'] = "神经网络"
}else if(dbo.value_type == 10){
d['O'] = "无方式"
}else{
d['O'] = ""
}
}
})
}
def findCommunityByEs(city,region, name){
def names = new HashSet()
def map = [:]
ElasticUtils.executeSqlQuery("select * from community where city='${city}' and region='${region}' and name='${name}' and is_new=1", { d->
map.put(d.source, d)
})
return map;
}
def addCcbPrice(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d->
@@ -114,6 +216,15 @@ class FjwPricePkCcbPrice {
})
}
def standRegion(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\写字楼入clickhouse.xlsx",{d->
if(!d['S']){
def newRegion = RegionData.formatRegion(d['A'], d['B'].replace("区区","区"))
d['X'] = newRegion
}
})
}
def run(){
def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size()
@@ -284,4 +395,78 @@ class FjwPricePkCcbPrice {
}
out.close();
}
def exportFjwPriceAndCcbPrice(){
def citys = ["杭州"]
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name"]);
println c +" "+districts.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-05','yyyy-MM-dd'))
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def rets = []
rets << ["城市","区域","名称","房价网价格","房价网计算方式","建行价格","建行计算方式","建行计算时间"]
for(def dist : districts ){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if(priceMap){
ret << priceMap.avg_price
if(priceMap.value_type == 1){
ret << "指导价"
}else if(priceMap.value_type == 2){
ret << "自身样本"
}else if(priceMap.value_type == 3){
ret << "第三方"
}else if(priceMap.value_type == 4){
ret << "交易数据"
}else if(priceMap.value_type == 5){
ret << "周边楼盘"
}else if(priceMap.value_type == 6){
ret << "推导"
}else if(priceMap.value_type == 9){
ret << "神经网络"
}else if(priceMap.value_type == 10){
ret << "无方式"
}else{
ret << ""
}
}else{
ret << ''
ret << ''
}
def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(c))
if(map){
ret << map.avg_price
ret << map.calc_type
ret << map.credit_value
ret << map.month
}
rets << ret
}
ExcelUtils.write("d:\\数据\\瑞丰\\杭州价格.xlsx",rets, c)
println c+" 导出完成"
}
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
}
@@ -224,7 +224,7 @@ class GuiDingPriceBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -102,7 +102,7 @@ class StagingDistrictBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -71,6 +71,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//
//})
def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
def allCitys = CcbCommonUtils.citys_all
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def c : allCitys){
def cityCode = cityCodeMap.get(c)
def count = null;
index++
for(def r : countCitys){
if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
count = r['B']
break
}
}
println index +" "+ c+" "+count
}
def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:]
@@ -632,7 +632,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -29,6 +30,9 @@ class BigdataCitypriceBatch extends BaseBatch{
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
}
@@ -64,7 +68,38 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_CITYPRICE 数据文件输出完成......"
println "txt 生成-> BIGDATA_CITYPRICE"
// saveToPgSqlCityPrice(rets)
}
def saveToPgSqlCityPrice(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
ps.toString()
}
}
println "end: "+new Date()
}
@@ -109,7 +144,37 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_DISTPRICE 数据文件输出完成......"
println "txt 生成-> BIGDATA_DISTPRICE"
// saveToPgSqlRegionPrice(rets)
}
def saveToPgSqlRegionPrice(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}
}
println "end: "+new Date()
}
def districtRun(citys, startDate, endDate, cityCodeDataMap){
@@ -140,9 +205,13 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "BIGDATA_COMMPRICE 数据文件输出完成......"
println "txt 生成->BIGDATA_COMMPRICE"
// saveToPgSqlDistrictPrice(rets)
}
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
@@ -172,6 +241,35 @@ class BigdataCitypriceBatch extends BaseBatch{
return rets;
}
def saveToPgSqlDistrictPrice(rets){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}
}
println "end: "+new Date()
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def sellDataType = 1
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
}
def calcMonthlyPrice(){
def citys = CommonUtils.findAllCity()
def citys = ["福州"]//CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
index++
def districts = []
if(city == "福州"){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList()
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
}
@@ -117,8 +117,8 @@ class DistrictSellStatBatch extends BaseBatch {
def rets = new CopyOnWriteArrayList()
districts.each{dist->
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name
@@ -202,8 +202,8 @@ class DistrictSellStatBatch extends BaseBatch {
} finally {
latch.countDown()
}
};
})
// };
// })
}
latch.await()
}
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
// task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -28,8 +28,8 @@ class CityStatExport {
def task = new CityStatExport();
// task.run()
// task.run1()
// task.regionStat()
task.stat()
task.regionStat()
// task.stat()
}
def stat(){
@@ -47,26 +47,26 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = ["太原","大同","临汾","朔州","忻州","晋中","阳泉","运城","长治","吕梁","晋城"]
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def ret = []
// ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"]
ret << ["城市","楼栋总量", "房号总量"]
ret << ["城市","小区总量"]
// ret << ["城市","楼栋总量", "房号总量"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
def total = districts.size()
def m = []
m << city
// m << total
m << total
// def tradeCount = findTrades(city, '2022-02-01');
// m << tradeCount
// def houseCount = findHouseCount(city, '2022-02-01')
// m << houseCount
def loudongCount = findLoudongCount(city)
m << loudongCount
def fanghaoCount = findFanghaoCount(city)
m << fanghaoCount
// def loudongCount = findLoudongCount(city)
// m << loudongCount
//
// def fanghaoCount = findFanghaoCount(city)
// m << fanghaoCount
// def priceCount = findDistrictPrice(city, null);
@@ -78,7 +78,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\山西省城市数据统计(楼栋房号).xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
}
@@ -96,12 +96,17 @@ class CityStatExport {
}
def regionStat(){
def city = "无锡"
def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def districts = CommonDatautils.findDistricts(city, r, null)
println r +" "+ districts.size()
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def ret = []
ret << ["城市","区域","小区总量"]
for(def city : citys){
def regions = CommonDatautils.getRegionListByCity(city)
for(def r : regions){
def districts = CommonDatautils.findDistricts(city, r, null)
println city +" "+ r +" "+ districts.size()
}
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县")
}
@@ -0,0 +1,28 @@
package com.fangjia.script.gj.district
import ch.qos.logback.core.db.dialect.DBUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
def opt = d['J']
if(opt && opt.contains("别名")){
def dist = CommonDatautils.findDistrictById(d['H'])
if(dist){
def alias = []
if(dist.alias){
alias = dist.alias
}
if(!alias.contains(d['C'])){
alias << d['C']
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
}
}
})
@@ -401,7 +401,7 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("重庆")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
@@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["重庆"]
def citys= ["福州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia") ){
cityMap.put(e.text(), href)
}
}
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "365淘房"//
def citys = findCrawlCity(source, 202211);
def source = "搜房"//
def citys = findCrawlCity(source, 202301);
for(def city : citys){
def count1 = findCount(source, city, 202211);
def count2 = findCount(source, city, 202212);
def count1 = findCount(source, city, 202301);
def count2 = findCount(source, city, 202302);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -26,7 +26,8 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["搜房":"搜房"]
static main(args) {
@@ -42,8 +43,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
def citys = ["凉山州"]//CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource, mdate)
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202212);
def count2 = findTotalCount(source, 202301);
def count1 = findTotalCount(source, 202301);
def count2 = findTotalCount(source, 202302);
println source +" "+count1 +" "+count2
}
}