From 08ff0e34fea0ec6b37eaf4491e040ded571e9873 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Mar 2023 17:09:48 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E5=AF=BC=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../BigdataCommunityBatch.groovy | 16 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 43 ++++ .../ccbevaluation/FjwPricePkCcbPrice.groovy | 187 +++++++++++++++++- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 17 ++ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../ronghe/BigdataCitypriceBatch.groovy | 104 +++++++++- .../fuzhou/DistrictSellStatBatch.groovy | 12 +- .../script/fuzhou/FzDataResultStat.groovy | 2 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/CityStatExport.groovy | 39 ++-- .../script/gj/district/AddAlias.groovy | 28 +++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 2 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 4 +- .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 6 +- .../TestCrawlTotalCount.groovy | 4 +- 20 files changed, 434 insertions(+), 50 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 90803d4ca..3418defb3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -120,7 +120,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 5cfbacaeb..0ba28a921 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,7 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -76,7 +77,9 @@ class BigdataCommunityBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" -// saveToPgSql(rows) + +// delPgData() + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } @@ -228,6 +231,11 @@ class BigdataCommunityBatch extends BaseBatch{ return ret; } + def delPgData(){ + sqlInstance.execute("delete from base.t0831_bigdata_community_gj_h") + println "base.t0831_bigdata_community_gj_h 数据已删除" + } + def saveToPgSql(rows){ for(def r : rows) { for (def i = 0; i < r.size(); i++) { @@ -236,7 +244,7 @@ class BigdataCommunityBatch extends BaseBatch{ } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null @@ -247,7 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(20000, sql){ ps-> + sqlInstance.withBatch(10, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 35c9912a8..e00fdeb41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -694,7 +694,7 @@ class BigdataDealBatch extends BaseBatch{ } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index b0407e1b5..4035c2270 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -558,6 +558,49 @@ class CcbCommonUtils { } + public static def findPgCcbDistrict(cityid){ + def sql = "select * from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}'"; + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + map.put(r.gujia_comm_id, data) + }); + return map + } + + public static def findTradesByMonth(city, startDate, endDate){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 4d8b7fa02..580fd9162 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -1,11 +1,14 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject import groovy.sql.Sql import org.apache.commons.lang.StringUtils @@ -23,6 +26,9 @@ class FjwPricePkCcbPrice { def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { def task = new FjwPricePkCcbPrice() // task.run() @@ -30,8 +36,48 @@ class FjwPricePkCcbPrice { // task.addFjwPrice() // task.addCcbPrice() - task.addCcbPriceFromGuided() +// task.addCcbPriceFromGuided() // task.addMarketPrice() + +// task.exportFjwPriceAndCcbPrice() +// task.addNewDistPrice() + +// task.addCcbPrice2() +// task.addCcbPrice3() + task.standRegion() + } + + def addCcbPrice2(){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d -> + if(d['R']){ + def map = findAvgPrice(d['R'], cityCodeMap.get(d['S'])) + if(map){ + d['AG'] = map.avg_price + d['AH'] = map.calc_type + d['AI'] = map.credit_value + d['AJ'] = map.month + } + } + }) + } + + def addCcbPrice3(){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\瑞丰\\金华、绍兴国信达查价_最新建行价格.xlsx", { d -> + if(d['A']){ + def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(dist){ + def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(d['A'])) + if(map){ + d['F'] = map.avg_price + d['G'] = map.calc_type + d['H'] = map.month + } + } + + } + }, 1) } def addMarketPrice(){ @@ -45,6 +91,62 @@ class FjwPricePkCcbPrice { }) } + def addNewDistPrice(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新楼盘添加指导价.xlsx",{d-> +// def map = findCommunityByEs(d['B'], d['C'], d['D']) +// if(map){ +// if(map.get("安居客-新房")){ +// def dbo = map.get("安居客-新房") +// d['F'] = dbo.avg_price +// d['G'] = dbo.open_date +// } +// if(map.get("贝壳找房-新房")){ +// def dbo = map.get("贝壳找房-新房") +// d['H'] = dbo.avg_price +// d['I'] = dbo.open_date +// }else if(map.get("楼盘网-新房")){ +// def dbo = map.get("楼盘网-新房") +// d['J'] = dbo.avg_price +// d['K'] = dbo.open_date +// } +// } + def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd')) +// def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-12','yyyy-MM-dd')) + if(dbo){ + d['N'] = dbo.avg_price + + if(dbo.value_type == 1){ + d['O'] = "指导价" + }else if(dbo.value_type == 2){ + d['O'] = "自身样本" + }else if(dbo.value_type == 3){ + d['O'] = "第三方" + }else if(dbo.value_type == 4){ + d['O'] = "交易数据" + }else if(dbo.value_type == 5){ + d['O'] = "周边楼盘" + }else if(dbo.value_type == 6){ + d['O'] = "推导" + }else if(dbo.value_type == 9){ + d['O'] = "神经网络" + }else if(dbo.value_type == 10){ + d['O'] = "无方式" + }else{ + d['O'] = "" + } + } + }) + } + + def findCommunityByEs(city,region, name){ + def names = new HashSet() + def map = [:] + ElasticUtils.executeSqlQuery("select * from community where city='${city}' and region='${region}' and name='${name}' and is_new=1", { d-> + map.put(d.source, d) + }) + return map; + } + def addCcbPrice(){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d-> @@ -114,6 +216,15 @@ class FjwPricePkCcbPrice { }) } + def standRegion(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\写字楼入clickhouse.xlsx",{d-> + if(!d['S']){ + def newRegion = RegionData.formatRegion(d['A'], d['B'].replace("区区","区")) + d['X'] = newRegion + } + }) + } + def run(){ def ccbPrices = getCommunityBasePriceGj(202208) println "ccb总量:"+ccbPrices.size() @@ -284,4 +395,78 @@ class FjwPricePkCcbPrice { } out.close(); } + + def exportFjwPriceAndCcbPrice(){ + def citys = ["杭州"] + for(def c : citys){ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name"]); + println c +" "+districts.size() + def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2023-03-05','yyyy-MM-dd')) + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def rets = [] + rets << ["城市","区域","名称","房价网价格","房价网计算方式","建行价格","建行计算方式","建行计算时间"] + for(def dist : districts ){ + def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name) + def ret = [] + ret << dist.city + ret << dist.region + ret << dist.name + if(priceMap){ + ret << priceMap.avg_price + if(priceMap.value_type == 1){ + ret << "指导价" + }else if(priceMap.value_type == 2){ + ret << "自身样本" + }else if(priceMap.value_type == 3){ + ret << "第三方" + }else if(priceMap.value_type == 4){ + ret << "交易数据" + }else if(priceMap.value_type == 5){ + ret << "周边楼盘" + }else if(priceMap.value_type == 6){ + ret << "推导" + }else if(priceMap.value_type == 9){ + ret << "神经网络" + }else if(priceMap.value_type == 10){ + ret << "无方式" + }else{ + ret << "" + } + }else{ + ret << '' + ret << '' + } + def map = findAvgPrice(dist._id.toString(), cityCodeMap.get(c)) + if(map){ + ret << map.avg_price + ret << map.calc_type + ret << map.credit_value + ret << map.month + } + rets << ret + } + ExcelUtils.write("d:\\数据\\瑞丰\\杭州价格.xlsx",rets, c) + println c+" 导出完成" + } + } + + + def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 30769569f..cc79fcf90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -224,7 +224,7 @@ class GuiDingPriceBatch extends BaseBatch{ } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 899243d11..cae04cf55 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -102,7 +102,7 @@ class StagingDistrictBatch extends BaseBatch{ } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index c2b8fcf3f..5474a893c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -71,6 +71,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // //}) +def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1) +def allCitys = CcbCommonUtils.citys_all +def cityCodeMap = CcbCommonUtils.findCityCodeValue() +def index = 0 +for(def c : allCitys){ + def cityCode = cityCodeMap.get(c) + def count = null; + index++ + for(def r : countCitys){ + if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){ + count = r['B'] + break + } + } + println index +" "+ c+" "+count +} + def findCcbPriceCredit(sqlInstance, cityid, commid){ def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 1d774446a..8091fe753 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -632,7 +632,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index e3b08e9e1..9a7d539e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject +import groovy.sql.Sql import org.apache.commons.lang.StringUtils import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder @@ -29,6 +30,9 @@ class BigdataCitypriceBatch extends BaseBatch{ def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCitypriceBatch(), args) } @@ -64,7 +68,38 @@ class BigdataCitypriceBatch extends BaseBatch{ } def headers = [] FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); - println "BIGDATA_CITYPRICE 数据文件输出完成......" + println "txt 生成-> BIGDATA_CITYPRICE" + +// saveToPgSqlCityPrice(rets) + } + + def saveToPgSqlCityPrice(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(1000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15]) + ps.toString() + } + } + + println "end: "+new Date() } @@ -109,7 +144,37 @@ class BigdataCitypriceBatch extends BaseBatch{ } def headers = [] FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); - println "BIGDATA_DISTPRICE 数据文件输出完成......" + println "txt 生成-> BIGDATA_DISTPRICE" +// saveToPgSqlRegionPrice(rets) + } + + def saveToPgSqlRegionPrice(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(1000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17]) + ps.toString() + } + } + + println "end: "+new Date() } def districtRun(citys, startDate, endDate, cityCodeDataMap){ @@ -140,9 +205,13 @@ class BigdataCitypriceBatch extends BaseBatch{ } def headers = [] FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); - println "BIGDATA_COMMPRICE 数据文件输出完成......" + println "txt 生成->BIGDATA_COMMPRICE" + +// saveToPgSqlDistrictPrice(rets) } + + def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def totalCount = new AtomicLong() def distCount = districts.size() @@ -172,6 +241,35 @@ class BigdataCitypriceBatch extends BaseBatch{ return rets; } + def saveToPgSqlDistrictPrice(rets){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(1000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17]) + ps.toString() + } + } + + println "end: "+new Date() + } + def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def sellDataType = 1 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 1b604d376..2e14d2f29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys = CommonUtils.findAllCity() + def citys = ["福州"]//CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) @@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch { index++ def districts = [] if(city == "福州"){ - districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() + districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList() }else{ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) } @@ -117,8 +117,8 @@ class DistrictSellStatBatch extends BaseBatch { def rets = new CopyOnWriteArrayList() districts.each{dist-> - fixedThreadPool.execute(new Runnable() { - void run() { +// fixedThreadPool.execute(new Runnable() { +// void run() { try { totalCount.addAndGet(1) def key = dist.region +"##" + dist.name @@ -202,8 +202,8 @@ class DistrictSellStatBatch extends BaseBatch { } finally { latch.countDown() } - }; - }) +// }; +// }) } latch.await() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 917ddfc5e..2da506d11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); - task.regionPriceTrendRun() +// task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 44e4dd54c..89347b833 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index 2e178125c..d6432197b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -28,8 +28,8 @@ class CityStatExport { def task = new CityStatExport(); // task.run() // task.run1() -// task.regionStat() - task.stat() + task.regionStat() +// task.stat() } def stat(){ @@ -47,26 +47,26 @@ class CityStatExport { // def citys = CommonDatautils.findAllCity(); - def citys = ["太原","大同","临汾","朔州","忻州","晋中","阳泉","运城","长治","吕梁","晋城"] + def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] def ret = [] - // ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"] - ret << ["城市","楼栋总量", "房号总量"] + ret << ["城市","小区总量"] +// ret << ["城市","楼栋总量", "房号总量"] for(def city : citys){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); def total = districts.size() def m = [] m << city -// m << total + m << total // def tradeCount = findTrades(city, '2022-02-01'); // m << tradeCount // def houseCount = findHouseCount(city, '2022-02-01') // m << houseCount - def loudongCount = findLoudongCount(city) - m << loudongCount - - def fanghaoCount = findFanghaoCount(city) - m << fanghaoCount +// def loudongCount = findLoudongCount(city) +// m << loudongCount +// +// def fanghaoCount = findFanghaoCount(city) +// m << fanghaoCount // def priceCount = findDistrictPrice(city, null); @@ -78,7 +78,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\数据\\山西省城市数据统计(楼栋房号).xlsx",ret,"数据") + ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据") } @@ -96,12 +96,17 @@ class CityStatExport { } def regionStat(){ - def city = "无锡" - def regions = CommonDatautils.getRegionListByCity(city) - for(def r : regions){ - def districts = CommonDatautils.findDistricts(city, r, null) - println r +" "+ districts.size() + def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def ret = [] + ret << ["城市","区域","小区总量"] + for(def city : citys){ + def regions = CommonDatautils.getRegionListByCity(city) + for(def r : regions){ + def districts = CommonDatautils.findDistricts(city, r, null) + println city +" "+ r +" "+ districts.size() + } } + ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy new file mode 100644 index 000000000..7b5b4df94 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.gj.district + +import ch.qos.logback.core.db.dialect.DBUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d-> + def opt = d['J'] + if(opt && opt.contains("别名")){ + def dist = CommonDatautils.findDistrictById(d['H']) + if(dist){ + def alias = [] + if(dist.alias){ + alias = dist.alias + } + if(!alias.contains(d['C'])){ + alias << d['C'] + def udata = new BasicDBObject() + udata.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + } + + } + } + +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 6c4e5732c..0bab915c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -401,7 +401,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("重庆")){ + if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 2bea3bcce..6ed04396a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["重庆"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia") ){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 62e2524e9..595fdeafb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "365淘房"// - def citys = findCrawlCity(source, 202211); + def source = "搜房"// + def citys = findCrawlCity(source, 202301); for(def city : citys){ - def count1 = findCount(source, city, 202211); - def count2 = findCount(source, city, 202212); + def count1 = findCount(source, city, 202301); + def count2 = findCount(source, city, 202302); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index dce00a5fa..ae3119029 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,8 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" - def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] +// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] + def esSources = ["搜房":"搜房"] static main(args) { @@ -42,8 +43,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = CommonUtils.findAllCity() - + def citys = ["凉山州"]//CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ def rets = findAllEsData(city, esSource, mgSource, mdate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 2d9309dae..5f9a10997 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202212); - def count2 = findTotalCount(source, 202301); + def count1 = findTotalCount(source, 202301); + def count2 = findTotalCount(source, 202302); println source +" "+count1 +" "+count2 } } From 30f69ca188635cec806c53d9c86409a715ca633e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Mar 2023 18:52:44 +0800 Subject: [PATCH 2/4] update --- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/FjwPricePkCcbPrice.groovy | 8 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 82 +++++++++---------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 0ba28a921..fcef31e5d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -255,7 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(10, sql){ ps-> + sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 4471922b8..9c836246f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -78,7 +78,7 @@ class BigdataForsaleBatch extends BaseBatch{ //目前查询库中是否已存在,用的是Oracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) - citys = ["迪庆州"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 580fd9162..df93b3dfc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -217,10 +217,10 @@ class FjwPricePkCcbPrice { } def standRegion(){ - ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\写字楼入clickhouse.xlsx",{d-> - if(!d['S']){ - def newRegion = RegionData.formatRegion(d['A'], d['B'].replace("区区","区")) - d['X'] = newRegion + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入3月20日.xlsx",{d-> + if(!d['J']){ + def newRegion = RegionData.formatRegion(d['A'], d['B']) + d['J'] = newRegion } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index cc79fcf90..7addbaec5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -43,49 +43,49 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) - def rows = [] - - CcbCommonUtils.deleteGuidedPrice(curDate) - for (def city : citys) { - def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - - //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-27") -// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") - if(prices){ - def distMap = getDistrictMap(city) - def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) - rows.addAll(rets) - println city + " " + rets.size() - } - - //添加搜房的参考价 -// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); -// def datas = findFangDatas(districts) -// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) -// rows.addAll(fangRets) - - //添加保亭文昌的指导价 -// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") -// def cityDataMap = [:] -// for(def d : cityDatas){ -// cityDataMap.put(d['A'], d['E']) +// def citys = CcbCommonUtils.getRunningCitys(args) +// def rows = [] +// +// CcbCommonUtils.deleteGuidedPrice(curDate) +// for (def city : citys) { +// def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// +// //添加自己库中的人工指导价 +// def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-27") +//// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") +// if(prices){ +// def distMap = getDistrictMap(city) +// def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) +// rows.addAll(rets) +// println city + " " + rets.size() // } -// def personDatas = findPersonDatas(districts, cityDataMap); -// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) -// rows.addAll(personRets) - - } - - def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); - - saveToPgSql(rows) +// +// //添加搜房的参考价 +//// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); +//// def datas = findFangDatas(districts) +//// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) +//// rows.addAll(fangRets) +// +// //添加保亭文昌的指导价 +//// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") +//// def cityDataMap = [:] +//// for(def d : cityDatas){ +//// cityDataMap.put(d['A'], d['E']) +//// } +//// def personDatas = findPersonDatas(districts, cityDataMap); +//// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) +//// rows.addAll(personRets) +// +// } +// +// def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] +// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); +// +// saveToPgSql(rows) //更新静态数据表 -// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } @@ -245,7 +245,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def updateVersion(def time){ - def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h"] + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"] for(def tableName : tables){ def sql = "update base."+tableName+" set p9_start_date='"+time+"'" pgsqlInstance.executeUpdate(sql) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8091fe753..94d42d3a3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202211 - def endDate = 202301 + def sDate= 202302//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202212 + def endDate = 202302 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From f7e2090b1e0bc006d002697963b0a48d3bb95386 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Mar 2023 19:00:24 +0800 Subject: [PATCH 3/4] update --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 9c836246f..507d204d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -58,7 +58,7 @@ class BigdataForsaleBatch extends BaseBatch{ } public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) @@ -153,8 +153,8 @@ class BigdataForsaleBatch extends BaseBatch{ for(def i=0;i Date: Mon, 20 Mar 2023 21:57:17 +0800 Subject: [PATCH 4/4] update --- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- .../com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 507d204d4..78e552ea5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -1106,11 +1106,11 @@ class BigdataForsaleBatch extends BaseBatch{ // 创建布隆过滤器对象 BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); // def houseIds = new HashSet() - pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h where urbn_id='533400'",{it-> + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h",{it-> filter.put(it.gujia_sell_id) }); - pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021 where urbn_id='533400'",{it-> + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021",{it-> filter.put(it.gujia_sell_id) }); return filter; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 0bab915c0..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list