diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataSynchPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataSynchPrice.groovy new file mode 100644 index 000000000..8d23f134e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataSynchPrice.groovy @@ -0,0 +1,192 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +/*** + * 同步上个月城市,区域,小区价格到base库中 + */ +class BigdataSynchPrice extends BaseBatch{ + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + + static main(args) { + runBatch(new BigdataSynchPrice(), args) + } + + + @Override + void process(Object args) { + + def citys = CcbCommonUtils.getRunningCitys(args) + +// synchCityPrice(citys) +// synchDistrictPrice(citys) + synchCommPirce(citys) + } + + + + def synchCityPrice(citys){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_city_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + + data << null + data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28" + data << null + data << '2999-12-31' + data << null + data << null + data << null + allRows << data + }); + } + + + + println "同步城市均价 start: "+new Date() + def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(500, sql){ ps-> + for(def r : allRows) { + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def synchDistrictPrice(citys){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'"; // + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + + data << null + data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28" + data << null + data << '2999-12-31' + data << null + data << null + data << null + allRows << data + }); + } + + println "同步区域均价 start: "+new Date() + def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(1000, sql){ ps-> + for(def r : allRows) { + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def synchCommPirce(citys){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + + data << null + data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28" + data << null + data << '2999-12-31' + data << null + data << null + data << null + allRows << data + }); + } + + + println "同步小区价格 start: "+new Date() + def sql = "insert into base.t0831_community_base_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(50000, sql){ ps-> + for(def r : allRows) { + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index c971794fd..9e365230a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202308.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 8634731d6..2003db863 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") -// if (citys.contains(e.text())){ + if (citys.contains(e.text())){ list << [e.text(), href] -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 22a51985b..ec1ad4acd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -587,7 +587,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 9d030c077..b57b42ef1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.ElasticUtils @@ -198,6 +199,7 @@ class GaoDePoiDataProcess extends BaseBatch{ if(gaodeMap && gaodeMap.district){ fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district) } + } if(!fjw_region && !retMap.fjw_district){ println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy index 6d64e17ed..9773313c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.xingyebank.Data import com.mongodb.BasicDBObject @@ -18,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong class PoiWash { def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) // DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL) @@ -34,9 +35,10 @@ class PoiWash { // task.updateId() // task.run() // task.synch() -// task.check() + task.check() - task.checkSpecialSymbols() + +// task.checkSpecialSymbols() } def updateId(){ @@ -102,10 +104,9 @@ class PoiWash { def checkSpecialSymbols(){ //"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市", - def citys = ["三明市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"] + def citys = ["重庆市"] for(def city : citys){ - def datas = findAllFieldPois(city) - specialSymbols(city, datas) + findAllFieldPois(city) println city +" 完成" } } @@ -256,7 +257,7 @@ class PoiWash { def findPois(def city){ def q = new BasicDBObject() q.put("city", city) -// q.put("name", '素匠泰茶') +// q.put("name", '教师公寓楼') def keys = new BasicDBObject() keys.put("_id", 1) @@ -268,7 +269,7 @@ class PoiWash { keys.put("address", 1) keys.put("fjw_city", 1) - def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000) + def dbcursor = MAPPOI_CONN.find(q).batchSize(10000) def rets = [] for(def d : dbcursor){ rets << d @@ -287,6 +288,16 @@ class PoiWash { def rets = [] for(def d : dbcursor){ rets << d + if(rets.size()>0 && rets.size()%100000==0){ + print "." + specialSymbols(city, rets) + rets.clear() + } + } + + if(rets.size()>0){ + specialSymbols(city, rets) + rets.clear() } return rets; } @@ -402,24 +413,77 @@ class PoiWash { try{ totalCount.addAndGet(1) - if(map.address){ - def newAddr = DataUtil.toDBCcase(map.address) - - if(newAddr.startsWith(";")){ - newAddr = newAddr.substring(1) + def udata = new BasicDBObject() + def cat = map.cat + if(cat && cat.contains(":")){ + def arrs = map.cat.split(":") + udata.put("type",arrs[0]) + if(arrs.size()>=2){ + udata.put("secondary_type",arrs[1]) } - if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){ - newAddr = "" + if(arrs.size()>=3){ + udata.put("small_type",arrs[2]) } - - if(map.address!=newAddr){ - updateTotalCount.addAndGet(1) - def udata = new BasicDBObject() - udata.put("address", newAddr) - MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata)) - println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city + if(arrs.size()>3){ + udata.put("cat",arrs[0]+";"+arrs[1]+";"+arrs[2]) } } + + if(map.bd_location){ + def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.bd_location.lon) + def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.bd_location.lat) + def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat] + udata.put("bd_location", new_bd_map) + } + + if(map.gps_location){ + def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.gps_location.lon) + def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.gps_location.lat) + def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat] + udata.put("gps_location", new_bd_map) + } + + if(map.address && (map.address.contains("(") || map.address.contains("("))){ + def s1 = null + def addr = map.address.replace("(", "(").replace(")", ")") + if(addr.contains("(") && addr.contains(")")){ + s1 = addr.substring(addr.indexOf("("),addr.indexOf(")")+1 ) + } + def new_addr = addr.replace(s1, "") + if(DataHandleUtils.isStandardRoad(new_addr)){ + udata.put("address", new_addr) + }else{ + if(DataHandleUtils.isStandardRoad(s1)){ + udata.put("address", s1.replace("(", "").replace(")", "")) + } + } + } + + if(udata){ + updateTotalCount.addAndGet(1) + MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata)) + println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city + } + + +// if(map.address){ +// def newAddr = DataUtil.toDBCcase(map.address) +// +// if(newAddr.startsWith(";")){ +// newAddr = newAddr.substring(1) +// } +// if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){ +// newAddr = "" +// } +// +// if(map.address!=newAddr){ +// updateTotalCount.addAndGet(1) +// def udata = new BasicDBObject() +// udata.put("address", newAddr) +// MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata)) +// println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city +// } +// } } catch (Exception e) { e.printStackTrace() } finally { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy index b3af27092..ddbde520f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy @@ -14,7 +14,7 @@ class ZheShangBank { } def sample(){ - def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt") + def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result-202309.txt") def rets = [] rets << ["时间","Msg", "城市", "区域", "名称","月份"] def map = [:] @@ -53,7 +53,7 @@ class ZheShangBank { // ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释") } def run(){ - def start = "2021-03-01" + def start = "2023-09-01" def rets = [] rets << ["月份","总调用量","成功量","失败量"] while (true){ @@ -61,18 +61,22 @@ class ZheShangBank { // def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null) // def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200) - def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null) - def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200) +// def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null) +// def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200) + + + def total = CommonDatautils.findLogsBySql("1007569", "/czbank/evaluation", start, end, null) + def successCount = CommonDatautils.findLogsBySql("1007569", "/czbank/evaluation", start, end, 200) def failCount = total - successCount println start[0..6] +" "+ total +" "+successCount +" "+ failCount rets << [start[0..6], total, successCount, failCount] start = end; - if(end == "2023-09-01"){ + if(end == "2023-10-01"){ break; } } - ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址") + ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计202309.xlsx", rets, "风险缓释估值接口") }