poi修改

This commit is contained in:
2023-10-17 11:41:57 +08:00
parent 2487baa532
commit 82c47163ea
7 changed files with 294 additions and 32 deletions
@@ -0,0 +1,192 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
/***
* 同步上个月城市,区域,小区价格到base库中
*/
class BigdataSynchPrice extends BaseBatch{
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new BigdataSynchPrice(), args)
}
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
// synchCityPrice(citys)
// synchDistrictPrice(citys)
synchCommPirce(citys)
}
def synchCityPrice(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_city_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步城市均价 start: "+new Date()
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(500, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def synchDistrictPrice(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'"; //
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gxda_admndiv_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步区域均价 start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def synchCommPirce(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步小区价格 start: "+new Date()
def sql = "insert into base.t0831_community_base_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202308.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
// if (citys.contains(e.text())){
if (citys.contains(e.text())){
list << [e.text(), href]
// }
}
}
}
return list
@@ -587,7 +587,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
}
}
@@ -6,6 +6,7 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
@@ -198,6 +199,7 @@ class GaoDePoiDataProcess extends BaseBatch{
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
}
}
if(!fjw_region && !retMap.fjw_district){
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
import com.mongodb.BasicDBObject
@@ -18,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
class PoiWash {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
// DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
@@ -34,9 +35,10 @@ class PoiWash {
// task.updateId()
// task.run()
// task.synch()
// task.check()
task.check()
task.checkSpecialSymbols()
// task.checkSpecialSymbols()
}
def updateId(){
@@ -102,10 +104,9 @@ class PoiWash {
def checkSpecialSymbols(){
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
def citys = ["三明市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
def citys = ["重庆市"]
for(def city : citys){
def datas = findAllFieldPois(city)
specialSymbols(city, datas)
findAllFieldPois(city)
println city +" 完成"
}
}
@@ -256,7 +257,7 @@ class PoiWash {
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("name", '素匠泰茶')
// q.put("name", '教师公寓楼')
def keys = new BasicDBObject()
keys.put("_id", 1)
@@ -268,7 +269,7 @@ class PoiWash {
keys.put("address", 1)
keys.put("fjw_city", 1)
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
def rets = []
for(def d : dbcursor){
rets << d
@@ -287,6 +288,16 @@ class PoiWash {
def rets = []
for(def d : dbcursor){
rets << d
if(rets.size()>0 && rets.size()%100000==0){
print "."
specialSymbols(city, rets)
rets.clear()
}
}
if(rets.size()>0){
specialSymbols(city, rets)
rets.clear()
}
return rets;
}
@@ -402,24 +413,77 @@ class PoiWash {
try{
totalCount.addAndGet(1)
if(map.address){
def newAddr = DataUtil.toDBCcase(map.address)
if(newAddr.startsWith(";")){
newAddr = newAddr.substring(1)
def udata = new BasicDBObject()
def cat = map.cat
if(cat && cat.contains(":")){
def arrs = map.cat.split(":")
udata.put("type",arrs[0])
if(arrs.size()>=2){
udata.put("secondary_type",arrs[1])
}
if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
newAddr = ""
if(arrs.size()>=3){
udata.put("small_type",arrs[2])
}
if(map.address!=newAddr){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", newAddr)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
if(arrs.size()>3){
udata.put("cat",arrs[0]+";"+arrs[1]+";"+arrs[2])
}
}
if(map.bd_location){
def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.bd_location.lon)
def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.bd_location.lat)
def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat]
udata.put("bd_location", new_bd_map)
}
if(map.gps_location){
def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.gps_location.lon)
def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.gps_location.lat)
def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat]
udata.put("gps_location", new_bd_map)
}
if(map.address && (map.address.contains("(") || map.address.contains("("))){
def s1 = null
def addr = map.address.replace("(", "(").replace(")", ")")
if(addr.contains("(") && addr.contains(")")){
s1 = addr.substring(addr.indexOf("("),addr.indexOf(")")+1 )
}
def new_addr = addr.replace(s1, "")
if(DataHandleUtils.isStandardRoad(new_addr)){
udata.put("address", new_addr)
}else{
if(DataHandleUtils.isStandardRoad(s1)){
udata.put("address", s1.replace("(", "").replace(")", ""))
}
}
}
if(udata){
updateTotalCount.addAndGet(1)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
}
// if(map.address){
// def newAddr = DataUtil.toDBCcase(map.address)
//
// if(newAddr.startsWith(";")){
// newAddr = newAddr.substring(1)
// }
// if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
// newAddr = ""
// }
//
// if(map.address!=newAddr){
// updateTotalCount.addAndGet(1)
// def udata = new BasicDBObject()
// udata.put("address", newAddr)
// MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
// println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
// }
// }
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -14,7 +14,7 @@ class ZheShangBank {
}
def sample(){
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt")
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result-202309.txt")
def rets = []
rets << ["时间","Msg", "城市", "区域", "名称","月份"]
def map = [:]
@@ -53,7 +53,7 @@ class ZheShangBank {
// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释")
}
def run(){
def start = "2021-03-01"
def start = "2023-09-01"
def rets = []
rets << ["月份","总调用量","成功量","失败量"]
while (true){
@@ -61,18 +61,22 @@ class ZheShangBank {
// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null)
// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200)
def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
// def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
// def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
def total = CommonDatautils.findLogsBySql("1007569", "/czbank/evaluation", start, end, null)
def successCount = CommonDatautils.findLogsBySql("1007569", "/czbank/evaluation", start, end, 200)
def failCount = total - successCount
println start[0..6] +" "+ total +" "+successCount +" "+ failCount
rets << [start[0..6], total, successCount, failCount]
start = end;
if(end == "2023-09-01"){
if(end == "2023-10-01"){
break;
}
}
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址")
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计202309.xlsx", rets, "风险缓释估值接口")
}