poi修改
This commit is contained in:
+192
@@ -0,0 +1,192 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
/***
|
||||
* 同步上个月城市,区域,小区价格到base库中
|
||||
*/
|
||||
class BigdataSynchPrice extends BaseBatch{
|
||||
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataSynchPrice(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// synchCityPrice(citys)
|
||||
// synchDistrictPrice(citys)
|
||||
synchCommPirce(citys)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def synchCityPrice(citys){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_city_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_trdpt_avprc_id
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujiabds_clc_yrmo_val
|
||||
data << r.gujia_bds_clc_avprc
|
||||
data << r.gujia_bds_urbn_avprc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
|
||||
data << null
|
||||
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
|
||||
data << null
|
||||
data << '2999-12-31'
|
||||
data << null
|
||||
data << null
|
||||
data << null
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
println "同步城市均价 start: "+new Date()
|
||||
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(500, sql){ ps->
|
||||
for(def r : allRows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def synchDistrictPrice(citys){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'"; //
|
||||
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_trdpt_avprc_id
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gxda_admndiv_nm
|
||||
data << r.gujiabds_clc_yrmo_val
|
||||
data << r.gujia_bds_clc_avprc
|
||||
data << r.gujia_bds_urbn_avprc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
|
||||
data << null
|
||||
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
|
||||
data << null
|
||||
data << '2999-12-31'
|
||||
data << null
|
||||
data << null
|
||||
data << null
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
println "同步区域均价 start: "+new Date()
|
||||
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(1000, sql){ ps->
|
||||
for(def r : allRows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def synchCommPirce(citys){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_bss_prc_id
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_bds_id
|
||||
data << r.comm_nm
|
||||
data << r.gujiabds_clc_yrmo_val
|
||||
data << r.gujia_bds_clc_avprc
|
||||
data << r.gujia_comm_basic_prc
|
||||
data << r.gujia_bds_clmtd
|
||||
data << r.gujiabds_fwd_samp_rto
|
||||
data << r.gujia_bds_mns_grd
|
||||
data << r.gujia_bds_cfdrg
|
||||
data << r.gujiabdsarndcmprccrlt
|
||||
data << r.gujia_bds_cfdc_dgr
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
|
||||
data << null
|
||||
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
|
||||
data << null
|
||||
data << '2999-12-31'
|
||||
data << null
|
||||
data << null
|
||||
data << null
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
println "同步小区价格 start: "+new Date()
|
||||
def sql = "insert into base.t0831_community_base_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : allRows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202308.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+2
-2
@@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
// if (citys.contains(e.text())){
|
||||
if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+1
-1
@@ -587,7 +587,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -6,6 +6,7 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
@@ -198,6 +199,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
|
||||
}
|
||||
|
||||
}
|
||||
if(!fjw_region && !retMap.fjw_district){
|
||||
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
|
||||
|
||||
+86
-22
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -18,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PoiWash {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
// DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
|
||||
@@ -34,9 +35,10 @@ class PoiWash {
|
||||
// task.updateId()
|
||||
// task.run()
|
||||
// task.synch()
|
||||
// task.check()
|
||||
task.check()
|
||||
|
||||
task.checkSpecialSymbols()
|
||||
|
||||
// task.checkSpecialSymbols()
|
||||
}
|
||||
|
||||
def updateId(){
|
||||
@@ -102,10 +104,9 @@ class PoiWash {
|
||||
|
||||
def checkSpecialSymbols(){
|
||||
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
|
||||
def citys = ["三明市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
def citys = ["重庆市"]
|
||||
for(def city : citys){
|
||||
def datas = findAllFieldPois(city)
|
||||
specialSymbols(city, datas)
|
||||
findAllFieldPois(city)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
@@ -256,7 +257,7 @@ class PoiWash {
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("name", '素匠泰茶')
|
||||
// q.put("name", '教师公寓楼')
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
@@ -268,7 +269,7 @@ class PoiWash {
|
||||
keys.put("address", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
@@ -287,6 +288,16 @@ class PoiWash {
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>0 && rets.size()%100000==0){
|
||||
print "."
|
||||
specialSymbols(city, rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
specialSymbols(city, rets)
|
||||
rets.clear()
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
@@ -402,24 +413,77 @@ class PoiWash {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
if(map.address){
|
||||
def newAddr = DataUtil.toDBCcase(map.address)
|
||||
|
||||
if(newAddr.startsWith(";")){
|
||||
newAddr = newAddr.substring(1)
|
||||
def udata = new BasicDBObject()
|
||||
def cat = map.cat
|
||||
if(cat && cat.contains(":")){
|
||||
def arrs = map.cat.split(":")
|
||||
udata.put("type",arrs[0])
|
||||
if(arrs.size()>=2){
|
||||
udata.put("secondary_type",arrs[1])
|
||||
}
|
||||
if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
|
||||
newAddr = ""
|
||||
if(arrs.size()>=3){
|
||||
udata.put("small_type",arrs[2])
|
||||
}
|
||||
|
||||
if(map.address!=newAddr){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", newAddr)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
if(arrs.size()>3){
|
||||
udata.put("cat",arrs[0]+";"+arrs[1]+";"+arrs[2])
|
||||
}
|
||||
}
|
||||
|
||||
if(map.bd_location){
|
||||
def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.bd_location.lon)
|
||||
def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.bd_location.lat)
|
||||
def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat]
|
||||
udata.put("bd_location", new_bd_map)
|
||||
}
|
||||
|
||||
if(map.gps_location){
|
||||
def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.gps_location.lon)
|
||||
def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.gps_location.lat)
|
||||
def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat]
|
||||
udata.put("gps_location", new_bd_map)
|
||||
}
|
||||
|
||||
if(map.address && (map.address.contains("(") || map.address.contains("("))){
|
||||
def s1 = null
|
||||
def addr = map.address.replace("(", "(").replace(")", ")")
|
||||
if(addr.contains("(") && addr.contains(")")){
|
||||
s1 = addr.substring(addr.indexOf("("),addr.indexOf(")")+1 )
|
||||
}
|
||||
def new_addr = addr.replace(s1, "")
|
||||
if(DataHandleUtils.isStandardRoad(new_addr)){
|
||||
udata.put("address", new_addr)
|
||||
}else{
|
||||
if(DataHandleUtils.isStandardRoad(s1)){
|
||||
udata.put("address", s1.replace("(", "").replace(")", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(udata){
|
||||
updateTotalCount.addAndGet(1)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
}
|
||||
|
||||
|
||||
// if(map.address){
|
||||
// def newAddr = DataUtil.toDBCcase(map.address)
|
||||
//
|
||||
// if(newAddr.startsWith(";")){
|
||||
// newAddr = newAddr.substring(1)
|
||||
// }
|
||||
// if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
|
||||
// newAddr = ""
|
||||
// }
|
||||
//
|
||||
// if(map.address!=newAddr){
|
||||
// updateTotalCount.addAndGet(1)
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("address", newAddr)
|
||||
// MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
// println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
// }
|
||||
// }
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
|
||||
+10
-6
@@ -14,7 +14,7 @@ class ZheShangBank {
|
||||
}
|
||||
|
||||
def sample(){
|
||||
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt")
|
||||
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result-202309.txt")
|
||||
def rets = []
|
||||
rets << ["时间","Msg", "城市", "区域", "名称","月份"]
|
||||
def map = [:]
|
||||
@@ -53,7 +53,7 @@ class ZheShangBank {
|
||||
// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释")
|
||||
}
|
||||
def run(){
|
||||
def start = "2021-03-01"
|
||||
def start = "2023-09-01"
|
||||
def rets = []
|
||||
rets << ["月份","总调用量","成功量","失败量"]
|
||||
while (true){
|
||||
@@ -61,18 +61,22 @@ class ZheShangBank {
|
||||
// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null)
|
||||
// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200)
|
||||
|
||||
def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
|
||||
def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
|
||||
// def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
|
||||
// def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
|
||||
|
||||
|
||||
def total = CommonDatautils.findLogsBySql("1007569", "/czbank/evaluation", start, end, null)
|
||||
def successCount = CommonDatautils.findLogsBySql("1007569", "/czbank/evaluation", start, end, 200)
|
||||
|
||||
def failCount = total - successCount
|
||||
println start[0..6] +" "+ total +" "+successCount +" "+ failCount
|
||||
rets << [start[0..6], total, successCount, failCount]
|
||||
start = end;
|
||||
if(end == "2023-09-01"){
|
||||
if(end == "2023-10-01"){
|
||||
break;
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址")
|
||||
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计202309.xlsx", rets, "风险缓释估值接口")
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user