Merge remote-tracking branch 'origin/script_gj' into script_gj
This commit is contained in:
+2
-2
@@ -3,6 +3,7 @@ package com.fangjia.script.ana
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -24,7 +25,7 @@ class FindStageDistrictBatch extends BaseBatch {
|
||||
@Override
|
||||
void process(Object args) {
|
||||
CommonUtils.findAllCity().each { def city ->
|
||||
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
|
||||
CommonDatautils.SEAWEED_COLL.find(DbUtils.buildSeaweedQuery2(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
|
||||
if (dbo.name =~ STAGE_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
|
||||
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
|
||||
@@ -40,7 +41,6 @@ class FindStageDistrictBatch extends BaseBatch {
|
||||
println(dbo)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+10
-8
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -72,19 +72,21 @@ for(def d : datas){
|
||||
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.area = d['F'].replace("平","").toString().toDouble()
|
||||
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
|
||||
if(d['I']){
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
|
||||
}
|
||||
|
||||
map.trade_date = d['J']
|
||||
// def trade_date = d['J']
|
||||
// if(trade_date){
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
// }else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
// else{
|
||||
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
|
||||
map.source = "深房家族"
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
|
||||
*/
|
||||
class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceSample()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
for(def i=1;i<19;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
|
||||
if(d['A'] && d['C']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
|
||||
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
println d['A'] +" 没查到"
|
||||
}
|
||||
}
|
||||
}, i)
|
||||
|
||||
println(i)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+12
-9
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def fangjiaDists = new HashSet()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
|
||||
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
for(def d : allDists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
for(def city : citys){
|
||||
|
||||
index ++
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
|
||||
println "加载建行小区量"
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}else{
|
||||
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
}
|
||||
|
||||
println "组装新数据"
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
|
||||
+32
-21
@@ -35,7 +35,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -62,20 +62,20 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
println objectId
|
||||
// def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
// def objectId = dateToObjectId(date)
|
||||
// println objectId
|
||||
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
//
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
def citys =CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
@@ -189,7 +189,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
latch.await();
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -385,7 +385,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
def data = distCodeMap.get(trade.match_region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
@@ -720,22 +720,33 @@ class BigdataDealBatch extends BaseBatch{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(region || (!region && ["澄迈"].contains(t.city))) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
def flag = false
|
||||
def address = null
|
||||
if(t.address){
|
||||
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
address = t.address
|
||||
}
|
||||
}
|
||||
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
|
||||
if(!rMap && !t.name.toString().endsWith("小区")){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
|
||||
}
|
||||
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else if (rMap && ["澄迈"].contains(rMap.city)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else{
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
@@ -752,7 +763,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
)
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
|
||||
+32
-11
@@ -7,18 +7,22 @@ import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
//"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照",
|
||||
public static def citys_fjw_all = [ "武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
//"乌鲁木齐",
|
||||
public static def citys_all = ["上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
|
||||
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
|
||||
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
|
||||
|
||||
@@ -233,7 +237,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -438,11 +442,14 @@ class CcbCommonUtils {
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
if(city){
|
||||
q.put("city", city);
|
||||
}
|
||||
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","新泾三村")
|
||||
// q.put("name","一米阳光一期")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -452,9 +459,9 @@ class CcbCommonUtils {
|
||||
}
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
def rets = new HashSet()
|
||||
def ids = new HashSet()
|
||||
|
||||
def ids = []
|
||||
for(def d : dbsour){
|
||||
if(kind){
|
||||
if(d.address){
|
||||
@@ -634,7 +641,8 @@ class CcbCommonUtils {
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', false))
|
||||
// q.put("district_name","中海寰宇天下")
|
||||
// q.put("district_name","新明胡同")
|
||||
// q.put("address","桂苑大道135号")
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
@@ -684,6 +692,17 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findGuiDingIdsByCity(){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
pgsqlInstance.eachRow("SELECT gujia_fsid FROM base.t0831_guided_price_gj_h",{it->
|
||||
filter.put(it.gujia_fsid)
|
||||
});
|
||||
return filter
|
||||
}
|
||||
|
||||
|
||||
public static def findDistGovPrice(def city){
|
||||
def ids = []
|
||||
pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it->
|
||||
@@ -720,6 +739,8 @@ class CcbCommonUtils {
|
||||
if(count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("trade_date", endDate)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("flag", "补漏变更成交时间")
|
||||
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -389,7 +389,7 @@ class DealPriceDataProcess {
|
||||
def deleteThirdPartyData(){
|
||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
+18
-4
@@ -47,16 +47,22 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def rows = []
|
||||
|
||||
CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
|
||||
def priceIdsfilter = CcbCommonUtils.findGuiDingIdsByCity()
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-27")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
//近3个月人工指导价
|
||||
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
|
||||
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
@@ -80,7 +86,8 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
println "人工指导价总计:"+rows.size()
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
@@ -142,11 +149,18 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
|
||||
if(priceIdsfilter.mightContain(ps._id.toString())){
|
||||
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
|
||||
continue
|
||||
}
|
||||
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
|
||||
+131
-22
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
|
||||
}
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
|
||||
// if(d['G']){
|
||||
// def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
// if(obj){
|
||||
// d['H'] = obj.name
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
////def url = "https://sh.anjuke.com"
|
||||
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
@@ -602,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
|
||||
////},0)
|
||||
//
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dbo = CommonDatautils.findDistrictById(d['D'])
|
||||
// def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dbo){
|
||||
// def udata = new BasicDBObject()
|
||||
//// udata.put("name_cat", 11)
|
||||
@@ -620,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dbo){
|
||||
def alias = null
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dbo.alias
|
||||
}
|
||||
if(!alias.contains(d['B'])){
|
||||
alias << d['B']
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dbo._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+"别名已存在"
|
||||
}
|
||||
|
||||
}else{
|
||||
println dbo._id.toString() +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
//for(def c : citys){
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
|
||||
@@ -1289,35 +1331,49 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['G']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['J']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['L'] = dist.status-dist.name_cat
|
||||
//// }
|
||||
//// }
|
||||
//// if(d['J'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
// if(dist){
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['H'] && d['G'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
|
||||
d['H'] = dist.name
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
//
|
||||
//// d['H'] = dist.name
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(file, { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['C'] = dist.city
|
||||
// d['D'] = dist.region
|
||||
// d['E'] = dist.name
|
||||
// def random = RandomUtil.random(-300,50)
|
||||
// d['F'] = random + d['B']
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
// if(d['B']){
|
||||
@@ -1338,6 +1394,56 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
//
|
||||
//})
|
||||
|
||||
//
|
||||
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
|
||||
//ExcelUtils.modify(filePath, { d ->
|
||||
// if(d['A']){
|
||||
//// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
|
||||
// if(list){
|
||||
// if(list.size()==1){
|
||||
// def dbo = list[0]
|
||||
// if(dbo){
|
||||
// def data = new BasicDBObject()
|
||||
// data.put("region", d['B'])
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }else{
|
||||
// println d['C'] +" 有多个"
|
||||
// }
|
||||
// }
|
||||
//// if(obj){
|
||||
//// def lng = d['G']
|
||||
//// def lat =d['H']
|
||||
////
|
||||
//// // 高德转百度
|
||||
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
//// def lng2 = baidu.x
|
||||
//// def lat2 = baidu.y
|
||||
////
|
||||
////
|
||||
//// def data = new BasicDBObject()
|
||||
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
|
||||
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
|
||||
//// data.put("lng", lng_new)
|
||||
//// data.put("lat", lat_new)
|
||||
////
|
||||
//// Long dLat2 = lat2 * 10000000000L
|
||||
//// Long dLng2 = lng2 * 10000000000L
|
||||
//// data.put("lat2", dLat2)
|
||||
//// data.put("lng2", dLng2)
|
||||
////
|
||||
////// data.put("status", 1)
|
||||
//// data.put("m_date", new Date())
|
||||
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
@@ -1439,6 +1545,9 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
//citys = CcbCommonUtils.citys_all
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
|
||||
if(d['G'] && d['G'] instanceof String){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
if(obj){
|
||||
d['H'] = obj.name
|
||||
}
|
||||
}
|
||||
})
|
||||
+22
-11
@@ -47,15 +47,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
// def gocPrices = CcbCommonUtils.findDistGovPrice("深圳")
|
||||
// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?"
|
||||
// pgsqlInstance.withBatch(500, sql){ ps->
|
||||
// for(def id : gocPrices){
|
||||
// ps.addBatch(id)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
@@ -265,6 +257,12 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
if(third.get("csjj_esf_price")){
|
||||
price_map.csjj_avg_price = third.get("csjj_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -311,6 +309,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "房产超市"
|
||||
}else if(k=="csjj_avg_price"){
|
||||
web = "城市居家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -528,6 +530,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
@@ -592,6 +595,14 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
|
||||
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
@@ -599,8 +610,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
if(value && value.size()>1){ //csjj_esf_price 国信达成交
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price && !value.csjj_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
|
||||
+69
-5
@@ -32,6 +32,10 @@ class TradeStatLog {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
@@ -40,8 +44,10 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
|
||||
task.exportTrade2()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -499,8 +505,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2023-08-01"
|
||||
def endDate = "2023-10-01"
|
||||
def startDate = "2024-04-01"
|
||||
def endDate = "2024-06-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
@@ -584,6 +590,36 @@ class TradeStatLog {
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
|
||||
def exportTrade2(){
|
||||
|
||||
// def startDate = "2024-06-01"
|
||||
// def endDate = "2024-07-01"
|
||||
def ids = findAllDealIds();
|
||||
println ids.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
|
||||
if(ids.contains(d['A'])){
|
||||
d['V'] = "供"
|
||||
}
|
||||
if(d['S'] && !d['W']){
|
||||
def dist = CommonDatautils.findDistrictById(d['S'])
|
||||
if(!dist.name_cat && dist.name_cat!=0){
|
||||
d['W'] = "name_cat过滤"
|
||||
}
|
||||
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
|
||||
d['W'] = "小区过滤"
|
||||
}
|
||||
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
|
||||
def map = findDealByCommId(d['B'], d['S'])
|
||||
def count = map.get(key)
|
||||
if(count>=2){
|
||||
d['W'] = "重复"
|
||||
}
|
||||
}
|
||||
println d['A']
|
||||
})
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
@@ -637,11 +673,39 @@ class TradeStatLog {
|
||||
return trades
|
||||
}
|
||||
|
||||
def findAllDealIds(){
|
||||
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
|
||||
def ids = []
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.gujia_bds_mdl_id)){
|
||||
ids << r.gujia_bds_mdl_id
|
||||
}
|
||||
})
|
||||
return ids
|
||||
}
|
||||
|
||||
def findDealByCommId(city, commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_id", commid)
|
||||
q.put("city", city)
|
||||
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findDealIds(city, startDate, endDate){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.dealid)){
|
||||
ids << r.dealid
|
||||
|
||||
|
||||
+110
-39
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def size = regionDatas.size()
|
||||
final CountDownLatch latch = new CountDownLatch(size)
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
for(def i = 0;i<regionDatas.size();i++){
|
||||
final def arr = regionDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
totalCount.addAndGet(1)
|
||||
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
|
||||
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
|
||||
if(row){
|
||||
rets << row
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// };
|
||||
// })
|
||||
}
|
||||
latch.await()
|
||||
// latch.await()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
|
||||
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def allDistRegionPreMap = [:]
|
||||
def allLastMonthMap = [:]
|
||||
def allLastYearMap = [:]
|
||||
def allCcbPriceMap = [:]
|
||||
def allCcbPriceCaseCountMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
|
||||
allDistricts.addAll(districts)
|
||||
|
||||
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
allPriceMap = allPriceMap + priceMap
|
||||
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
// allPriceMap = allPriceMap + priceMap
|
||||
|
||||
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
|
||||
allRentMap = allRentMap + rentMap
|
||||
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
|
||||
allDealCountMap = allDealCountMap + dealCountMap
|
||||
|
||||
|
||||
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
|
||||
|
||||
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceMap = allCcbPriceMap + datasMap
|
||||
|
||||
|
||||
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
|
||||
|
||||
def cityRankMap = findCommunityCityRank(datas)
|
||||
allCityRankMap = allCityRankMap + cityRankMap
|
||||
|
||||
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
allLastYearMap = allLastYearMap + lastYearMonthMap
|
||||
}
|
||||
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(cityLists){
|
||||
rets.addAll(cityLists)
|
||||
}
|
||||
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def totalCount = new AtomicLong()
|
||||
def distCount = districts.size()
|
||||
final CountDownLatch latch = new CountDownLatch(distCount)
|
||||
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(row){
|
||||
rets << row
|
||||
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
|
||||
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def sellDataType = 1
|
||||
def rentDataType = 1
|
||||
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
//房源查询是否按建行均价筛选房源
|
||||
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
def sell_avg_price = null
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
def sell_count = 0
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
|
||||
|
||||
if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
|
||||
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
}
|
||||
}
|
||||
if(sell_count){
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(sellMap){
|
||||
sell_count = sellMap.avg_price.size()
|
||||
if(sell_count>3){
|
||||
sell_avg_price = avgValue(sellMap.avg_price)
|
||||
if(sellMap){
|
||||
sell_avg_area = avgValue(sellMap.area)
|
||||
sell_avg_total = avgValue(sellMap.total_price)
|
||||
}else{
|
||||
sell_count = 0
|
||||
sellDataType = null
|
||||
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// if(sell_count>3){
|
||||
// if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def rent_avg_price = null
|
||||
def rent_avg_area = null
|
||||
def rent_avg_total = null
|
||||
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
rentDataType = null
|
||||
}
|
||||
}
|
||||
if(!sell_avg_price){
|
||||
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
|
||||
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
|
||||
// if(!sell_avg_price){
|
||||
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
// }
|
||||
if(!rent_avg_price){
|
||||
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
|
||||
def avgPrice = null
|
||||
if(region){
|
||||
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
|
||||
}else{
|
||||
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
|
||||
}
|
||||
|
||||
|
||||
if (sell_count < 10) {
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
sellStartDate = startDate
|
||||
}
|
||||
if (sell_count) {
|
||||
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
|
||||
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
|
||||
}
|
||||
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return datas;
|
||||
}
|
||||
|
||||
def findCommunityPriceByCityMap(cityCode, month){
|
||||
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCommunityPriceCountByCityMap(cityCode, month){
|
||||
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
if(r.gujiabdsrt3mlgcasenum){
|
||||
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
|
||||
}
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCcbCityPrice(cityCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def avgPrice = null
|
||||
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return regionPriceMap;
|
||||
}
|
||||
|
||||
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
|
||||
def price = null
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
price = r.gujia_bds_clc_avprc
|
||||
})
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
def districtCityPrictRate(cityPrice, datas){
|
||||
def distCityRateMap = [:]
|
||||
|
||||
+38
-38
@@ -146,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataCommunityGJRH(){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -221,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -253,7 +253,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -295,7 +295,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -327,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -356,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -384,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -422,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
});
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -452,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -485,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -517,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -552,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -576,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
@@ -676,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -716,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -987,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -998,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildingFJW(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1072,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1084,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportThirdPartyPrice2(price_date, new_dists) {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1132,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgThirdPartyPrice2(price_date, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1215,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1261,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1304,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1319,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1393,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1460,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1501,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1537,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1570,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1609,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1652,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1689,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCityPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1737,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataDistPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1784,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCommPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1845,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1856,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataForsale(publishDate, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1913,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgBigdataForsale(publishDate,endDate, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -2098,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
|
||||
+311
-35
@@ -13,11 +13,13 @@ import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
@@ -117,6 +119,8 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -274,7 +278,7 @@ class CommonDatautils {
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","泰瑞家园")
|
||||
// q.put("name","托斯卡纳庄园")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -283,7 +287,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -1205,6 +1209,13 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findAllCount(String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", cat);
|
||||
q.put("status", 0)
|
||||
return SEAWEED_MONGO_COLL.count(q)
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictsByName(String city, String name, String cat){
|
||||
|
||||
@@ -1843,39 +1854,6 @@ class CommonDatautils {
|
||||
}
|
||||
return priceMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend, cat){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
|
||||
if(cat == "district"){
|
||||
BasicDBObject queryObj = new BasicDBObject();
|
||||
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
|
||||
queryCondition.put("estate_type", queryObj);
|
||||
}else if(cat == "office"){
|
||||
queryCondition.put("estate_type", "写字楼");
|
||||
}else if(cat == "shop"){
|
||||
queryCondition.put("estate_type", "商铺");
|
||||
}
|
||||
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
public static def queryFirstDistrictPrice(city, region, name){
|
||||
@@ -1914,6 +1892,39 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend, cat){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
|
||||
if(cat == "district"){
|
||||
BasicDBObject queryObj = new BasicDBObject();
|
||||
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
|
||||
queryCondition.put("estate_type", queryObj);
|
||||
}else if(cat == "office"){
|
||||
queryCondition.put("estate_type", "写字楼");
|
||||
}else if(cat == "shop"){
|
||||
queryCondition.put("estate_type", "商铺");
|
||||
}
|
||||
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
@@ -4571,6 +4582,28 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def findAllTownRanges(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("town", 1)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def lists = townMap.get(d.city)
|
||||
if(!lists){
|
||||
lists = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.region
|
||||
t.town = d.town
|
||||
lists << t
|
||||
townMap.put(d.city, lists)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
@@ -4605,6 +4638,148 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryFullNameByCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","城市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = RegionData.formatCity(d.city)
|
||||
if(city){
|
||||
cityMap.put(city, d)
|
||||
}else{
|
||||
//println d.city +" 格式化不出"
|
||||
}
|
||||
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
static def queryDistNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
keys.put("district_short",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.mzb_district_name){
|
||||
def regions = regionMap.get(d.city)
|
||||
if(!regions){
|
||||
regions = []
|
||||
}
|
||||
regions << d
|
||||
regionMap.put(d.city, regions)
|
||||
}else{
|
||||
println d.province+" "+d.district +" 格式化不出民政部区域"
|
||||
}
|
||||
}
|
||||
|
||||
println "区域加载完成"
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryCommunityByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "乡镇街道 加载完成"
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.district
|
||||
t.town = d.street
|
||||
list << t
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -4632,6 +4807,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryDistNameCityByMapping(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.district_short && d.city_short){
|
||||
retMap.put(d.city_short+"#"+d.district_short, d)
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -4675,7 +4865,93 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
def road = ""
|
||||
def name = ""
|
||||
def type = ""
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if(addressComponent.streetNumber){
|
||||
if(addressComponent.streetNumber.street){
|
||||
road = addressComponent.streetNumber.street
|
||||
}
|
||||
if(addressComponent.streetNumber.number){
|
||||
road = road + addressComponent.streetNumber.number
|
||||
}
|
||||
}
|
||||
|
||||
if(addressComponent.building){
|
||||
if(addressComponent.building.name){
|
||||
name = addressComponent.building.name
|
||||
}
|
||||
if(addressComponent.building.type){
|
||||
type = addressComponent.building.type
|
||||
}
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
map.road = road
|
||||
map.type = type
|
||||
map.name = name
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
static def saveOptLog(city, region, name, cat, opt, user_name){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category",cat);
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", opt)
|
||||
obj.put("user_name", user_name)
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+5
-5
@@ -264,12 +264,12 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(city == "¸£ÖÝ"){
|
||||
if(d.avg_price_2b && d.value_type!=1){
|
||||
d.avg_price = d.avg_price_2b
|
||||
if(!priceMap.containsKey(d.region +"##"+d.district_name+"##"+d.estate_type)){
|
||||
if(city == "¸£ÖÝ"){
|
||||
if(d.avg_price_2b && d.value_type!=1){
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
|
||||
priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GreenBuild {
|
||||
|
||||
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new GreenBuild()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAll()
|
||||
def rets = match(datas)
|
||||
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
|
||||
}
|
||||
|
||||
def match(datas){
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
|
||||
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
data.match_id = resultMap.get("id").toString()
|
||||
data.match_city = resultMap.get("city")
|
||||
data.match_district = resultMap.get("district")
|
||||
data.match_name = resultMap.get("name")
|
||||
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
|
||||
}else{
|
||||
rets << [data._id.toString(), data.city, data.district, data.name]
|
||||
}
|
||||
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await();
|
||||
return rets
|
||||
}
|
||||
|
||||
def findAll(){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -13,12 +13,18 @@ import net.sf.json.groovy.JsonSlurper
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
|
||||
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
}
|
||||
if(suggestName){
|
||||
url = url +"&suggestName="+suggestName
|
||||
}
|
||||
if(suggestDistrict){
|
||||
url = url +"&suggestDistrict="+suggestDistrict
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
|
||||
@@ -12,8 +12,8 @@ class OpenUserBatch {
|
||||
def task = new OpenUserBatch();
|
||||
// task.run();
|
||||
// task.test();
|
||||
// task.addShenZhenFjtUser();
|
||||
task.addFujianFjtUser()
|
||||
task.addShenZhenFjtUser();
|
||||
// task.addFujianFjtUser()
|
||||
}
|
||||
|
||||
def test(){
|
||||
@@ -46,20 +46,20 @@ class OpenUserBatch {
|
||||
}
|
||||
|
||||
def addShenZhenFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252608 + i;
|
||||
def id = 252893 + i;
|
||||
BigDecimal bd = new BigDecimal(data[3].toString());
|
||||
String mobile = bd.toPlainString();
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
+247
File diff suppressed because one or more lines are too long
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.gson.JsonObject
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class BaiduBaikeTown {
|
||||
|
||||
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new BaiduBaikeTown()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
|
||||
if(!row['E'] && row['A']=="郑州经济技术开发区"){
|
||||
def q = new BasicDBObject()
|
||||
q.put("desc",new BasicDBObject('$ne', 'null'))
|
||||
q.put("name",row['A'])
|
||||
def d = dzb_COLL.findOne(q)
|
||||
def townMap = parse(d.desc, d.name)
|
||||
row['D'] = townMap.town_str1
|
||||
row['F'] = townMap.town_str2
|
||||
|
||||
// def ret = []
|
||||
// ret << ["名称", "基本信息", "目录", "全文"]
|
||||
// for(def d : dbcursor){
|
||||
// def townMap = parse(d.desc, d.name)
|
||||
// if(townMap.town){
|
||||
// println d.name+" "+townMap.town
|
||||
// }
|
||||
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
|
||||
}
|
||||
|
||||
def parse(desc, name){
|
||||
def map = [:]
|
||||
if(desc){
|
||||
def doc = Jsoup.parse(desc)
|
||||
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
|
||||
|
||||
def s = doc.toString()
|
||||
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
|
||||
def jsonObj = JSONObject.fromObject(dataStr)
|
||||
|
||||
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
|
||||
def towns = []
|
||||
if(jsonObj && jsonObj.catalog){
|
||||
def catalogs = jsonObj.catalog
|
||||
def index = null
|
||||
for(def log : catalogs){
|
||||
def curIndex = log.index
|
||||
if(log.title=="行政区划"){
|
||||
index = log.index
|
||||
}
|
||||
if(index && curIndex.toString().startsWith(index+"-")){
|
||||
towns << log.title
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
map.town_str1 = str
|
||||
if(town){
|
||||
map.town = town
|
||||
}else{
|
||||
map.town = ""
|
||||
}
|
||||
|
||||
if(towns){
|
||||
map.towns = towns.join(",")
|
||||
}else{
|
||||
map.towns = ""
|
||||
}
|
||||
|
||||
|
||||
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
|
||||
map.town_str2 = str2
|
||||
return map
|
||||
}
|
||||
map.town_str1 = ""
|
||||
map.town_str2 = ""
|
||||
map.town = ""
|
||||
map.towns = ""
|
||||
return map
|
||||
}
|
||||
}
|
||||
+467
-69
@@ -3,8 +3,11 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -15,21 +18,28 @@ import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DevProcess {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL)
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
// task.devSplitRoad()
|
||||
task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
@@ -38,17 +48,33 @@ class DevProcess {
|
||||
// task.updateCityAndDist()
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
task.matchDevAndAddAddress()
|
||||
// task.addDevAddres3()
|
||||
// task.matchDevAndAddAddress()
|
||||
|
||||
// task.stat()
|
||||
// task.addNewDev()
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
// task.synchDevTo100()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
// task.addDevShape()//添加开发区围栏
|
||||
// task.matchDevTownship()//添加乡镇街道
|
||||
|
||||
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
|
||||
|
||||
// task.matchNewCrawlDev()
|
||||
|
||||
// println DateUtils.add("2024-7-24", 7);
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
// task.checkTown()
|
||||
|
||||
// task.repeatSave()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -166,69 +192,177 @@ class DevProcess {
|
||||
}
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
def i = 0;
|
||||
while(i<1){
|
||||
def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\浙江-湖州.xlsx"
|
||||
def findDev( name){
|
||||
def obj = findDevByName(name)
|
||||
if(!obj){
|
||||
obj = findDevByAlias(name)
|
||||
if(obj){
|
||||
obj.match_way = "别名"
|
||||
}
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
/*if(d['C'] && !d['V']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['V'] = jsonObject.get("province")
|
||||
}
|
||||
}else{
|
||||
obj.match_way = "相同"
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['W'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['X'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['Y'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['Z'] = jsonObject.get("streetNumber")
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias1 = null
|
||||
if(!d.alias){
|
||||
alias1 = []
|
||||
}else{
|
||||
alias1 = d.alias
|
||||
}
|
||||
if(!alias1.contains(d.name)){
|
||||
alias1 << d.name
|
||||
}
|
||||
if(alias1){
|
||||
try {
|
||||
for(def n : alias1){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}*/
|
||||
if(d['C'] && !d['J']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
if(jsonObject){
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
}catch(exception){
|
||||
println alias1
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
def i = 1;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['C'] && !d['F']){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||
def devName = jsonObject1.get("dev")
|
||||
def obj = findDev(devName)
|
||||
d['F'] = devName
|
||||
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
|
||||
d['S'] = jsonObject1.detail.dev_confi
|
||||
}
|
||||
if (obj) {
|
||||
d['O'] = obj._id.toString()
|
||||
d['P'] = obj.name
|
||||
d['U'] = obj.match_way
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
|
||||
d['O'] = jsonObject.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
if(d['E'] && !d['M']) {
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
d['G'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
|
||||
d['H'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
|
||||
d['I'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
|
||||
d['J'] = jsonObject.get("township")
|
||||
}
|
||||
|
||||
if (jsonObject && street && streetNumber && street != "null") {
|
||||
d['K'] = jsonObject.get("street")
|
||||
def road = jsonObject.get("street")
|
||||
if (streetNumber != 'null') {
|
||||
d['L'] = jsonObject.get("streetNumber")
|
||||
road = road + jsonObject.get("streetNumber")
|
||||
} else {
|
||||
d['N'] = "信息不完整"
|
||||
}
|
||||
|
||||
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
|
||||
d['N'] = "原地址不含路"
|
||||
}
|
||||
|
||||
if (!d['N']) {
|
||||
def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$')
|
||||
if (!flag) {
|
||||
d['N'] = "路非法"
|
||||
}
|
||||
def roadsize = jsonObject.get("street").size()
|
||||
if (!d['N'] && (roadsize < 3 || roadsize > 7)) {
|
||||
d['N'] = "路太长|路太短"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['N'] = "无地址"
|
||||
}
|
||||
|
||||
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
|
||||
def devName = jsonObject.get("dev")
|
||||
def obj = findDev(devName)
|
||||
d['M'] = devName
|
||||
if (obj) {
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
d['V'] = obj.match_way
|
||||
}
|
||||
if(jsonObject.containsKey("detail") && jsonObject.detail.dev_confi!='null'){
|
||||
d['T'] = jsonObject.detail.dev_confi
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
i++
|
||||
println fileName+" 完成"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -295,14 +429,27 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
|
||||
if(!d['P'] && d['R']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
|
||||
if(!d['I'] && d['J']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['R'])
|
||||
// q.put("name", d['J'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['J']]))
|
||||
def name = d['J'].replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['P'] = obj._id.toString()
|
||||
d['Q'] = obj.name
|
||||
d['I'] = "库中有开发区-相似2"
|
||||
d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
// d['D'] = obj.name
|
||||
// if(obj.name == d['A']){
|
||||
// d['E'] = "相同"
|
||||
// }else{
|
||||
// d['E'] = "包含"
|
||||
// }
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
}
|
||||
}
|
||||
},0);
|
||||
@@ -469,6 +616,38 @@ class DevProcess {
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
|
||||
def addDevAddres3(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
|
||||
if(d['A']){
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
d['C'] = obj._id.toString()
|
||||
|
||||
def q2 = new BasicDBObject()
|
||||
q2.put("name", d['A'])
|
||||
q2.put("address", d['B'])
|
||||
q2.put("type","地址")
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q2)
|
||||
if(!sameObj){
|
||||
index++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
udata.put("city", obj.city)
|
||||
udata.put("district", obj.district)
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address", d['B'])
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
println index
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
def total = 0
|
||||
@@ -593,27 +772,27 @@ class DevProcess {
|
||||
});
|
||||
}
|
||||
|
||||
def synchDevTo43(){
|
||||
def dbcursor = development_zone_CONN.find()
|
||||
def synchDevTo100(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
|
||||
def rets = []
|
||||
for(def r : dbcursor){
|
||||
rets << r
|
||||
}
|
||||
DEVELOPMENT_ZONE_43_COLL.insert(rets)
|
||||
development_zone_CONN.insert(rets)
|
||||
}
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
|
||||
if(d['R']){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
|
||||
if(d['E']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['R'])
|
||||
q.put("name", d['E'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
|
||||
d['P'] = obj._id.toString()
|
||||
d['Q'] = obj.name
|
||||
def address = d['L']+d['M']
|
||||
d['F'] = obj._id.toString()
|
||||
d['G'] = obj.name
|
||||
def address = d['D']
|
||||
q.put("address", address)
|
||||
// def street = d['G']
|
||||
// q.put("township", street)
|
||||
@@ -631,6 +810,7 @@ class DevProcess {
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address",address)
|
||||
udata.put("c_date", new Date())
|
||||
// udata.put("type", "乡镇街道")
|
||||
// udata.put("township", street)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
@@ -641,6 +821,44 @@ class DevProcess {
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchDevTownship(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
|
||||
if(d['D']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['D'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['F'] = obj._id.toString()
|
||||
def street = d['E']
|
||||
q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
if(obj.city){
|
||||
udata.put("city", obj.city)
|
||||
}
|
||||
if(obj.district){
|
||||
udata.put("district", obj.district)
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "乡镇街道")
|
||||
udata.put("township", street)
|
||||
|
||||
// udata.put("type", "区域")
|
||||
// udata.put("dev_district", street)
|
||||
udata.put("c_date",new Date())
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+street
|
||||
}
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def addAlias(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
|
||||
def q = new BasicDBObject()
|
||||
@@ -665,4 +883,184 @@ class DevProcess {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevShape(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d->
|
||||
if(d['W']){
|
||||
def s = d['S']
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
Map<String, Object> map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", map)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevStreet(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
|
||||
if(d['J']=="是"){
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道"))
|
||||
for(def obj : dbcursor){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province" , obj.province)
|
||||
o.put("city" , obj.city)
|
||||
o.put("district" , obj.mzb_district_name)
|
||||
o.put("name" , d['E'])
|
||||
o.put("type" , "乡镇街道")
|
||||
o.put("township" , obj.street)
|
||||
DEV_MAPPING_CONN.save(o)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def matchNewCrawlDev(){
|
||||
def query = new BasicDBObject()
|
||||
query.put("match_id", new BasicDBObject('$exists', false))
|
||||
def list = kfq_COLL.find(query).toList()
|
||||
for(def d : list){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d.name)
|
||||
q.put('alias', new BasicDBObject('$in',[d.name]))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_id", obj._id.toString())
|
||||
udata.put("match_name", obj.name)
|
||||
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addKfc(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
|
||||
if(d['I']){
|
||||
if(d['I'] == "添加"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("create_time" , new Date())
|
||||
udata.put("industry",d['D'])
|
||||
if(d['E'].contains("国家级")){
|
||||
udata.put("level" ,"国家级")
|
||||
}else if(d['E'].contains("省级")){
|
||||
udata.put("level" ,"省级")
|
||||
}
|
||||
udata.put("name" ,d['F'])
|
||||
udata.put("province" ,d['G'])
|
||||
udata.put("type" ,d['H'])
|
||||
udata.put("city" ,d['B'])
|
||||
udata.put("district" ,d['J'])
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['I'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def alias = []
|
||||
if(obj.alias){
|
||||
alias = obj.alias
|
||||
}
|
||||
if(!alias.contains(d['F'])){
|
||||
alias << d['F']
|
||||
}
|
||||
if(alias && alias.size()!=obj.alias.size()){
|
||||
def updateData = new BasicDBObject()
|
||||
updateData.put("alias", alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def devToPark(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
|
||||
def rets = []
|
||||
rets << ["开发区名称", "下辖街道", "城市", "区域"]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def arrs = d['E'].split("、")
|
||||
def q = new BasicDBObject().append("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
for(def park : arrs){
|
||||
rets << [d['A'], park, obj.city, obj.district]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
|
||||
}
|
||||
|
||||
|
||||
def checkTown(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
|
||||
|
||||
if(!d['E']){
|
||||
if(d['A']){
|
||||
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
|
||||
// if(jsonObject1) {
|
||||
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
// def township = jsonObject1.get("township")
|
||||
// def district = jsonObject1.get("district")
|
||||
// d['E'] = "接口"
|
||||
// if(district && district!='null'){
|
||||
// d['F'] = district
|
||||
// }
|
||||
// if(township && township!='null'){
|
||||
// d['G'] = township
|
||||
// }
|
||||
//
|
||||
// d['H'] = jsonObject1.get("townCode")
|
||||
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
// d['I'] = jsonObject1.get("provinceCode")
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['C'])
|
||||
// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
// q.put("street", d['B'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['B']]))
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = "可入库-忽略区域模糊2"
|
||||
d['F'] = obj.district
|
||||
d['G'] = obj.street
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def repeatSave(){
|
||||
def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject())
|
||||
def list = []
|
||||
for(def map : dbcursor){
|
||||
def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"]
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(columns.contains(k)){
|
||||
if(v instanceof String){
|
||||
if(k=="_id"){
|
||||
udata.put("_id", new ObjectId(v))
|
||||
}else{
|
||||
udata.put(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
list << udata
|
||||
}
|
||||
DEV_MAPPING_CONN2.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
class DevUpdateTestBatch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
// task.addDevAddress(path)
|
||||
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDev(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['I'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(!obj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", d['B'])
|
||||
o.put("city", d['C'])
|
||||
o.put("district", d['D'])
|
||||
o.put("name", d['A'])
|
||||
o.put("level", d['E'])
|
||||
o.put("type", d['F'])
|
||||
|
||||
if(d['G']){
|
||||
o.put("township", d['G'])
|
||||
}
|
||||
if(d['H']){
|
||||
def arr = d['H'].split(",")
|
||||
if(arr){
|
||||
o.put("alias", arr)
|
||||
}
|
||||
}
|
||||
def id = addMongoDB(o)
|
||||
o.remove("_id")
|
||||
o.id = id.toString()
|
||||
o.remove("create_time")
|
||||
addEs(o, "192.168.0.89", "development_zone")
|
||||
addEs(o, "192.168.0.52", "development_zone")
|
||||
d['I'] = "已完成"
|
||||
}else{
|
||||
println d['A'] +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addMongoDB(o){
|
||||
o.put("create_time", new Date())
|
||||
development_zone_100_CONN.save(o)
|
||||
DEVELOPMENT_ZONE_43_COLL.save(o)
|
||||
return o._id
|
||||
}
|
||||
|
||||
def addEs(map, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addDevAddress(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['D'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
q.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
q.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
q.put("park", d['C'])
|
||||
}
|
||||
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
|
||||
if(!mappingObj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", obj.province)
|
||||
o.put("city", obj.city)
|
||||
o.put("district", obj.district)
|
||||
o.put("name", obj.name)
|
||||
o.put("type", d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
o.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
o.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
o.put("park", d['C'])
|
||||
}
|
||||
def date = new Date()
|
||||
o.put("c_date", date)
|
||||
o.put("version", date.getTime())
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
// o.id = o._id.toString()
|
||||
// o.remove("_id")
|
||||
// o.remove("c_date")
|
||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
||||
// try{
|
||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
||||
// }catch(Exception e){
|
||||
// println "52 es error: "+e.printStackTrace()
|
||||
// }
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
d['D'] = "重复"
|
||||
println d['A'] +" "+ d['C']+" 重复"
|
||||
}
|
||||
}else{
|
||||
d['D'] = "不存在"
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
def updateHasGov(province, city, district, street, hasGov){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", province)
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("street", street)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
//修改43
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
|
||||
|
||||
//修改89 administrative_mapping
|
||||
update89EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
//修改89 administrative_mapping
|
||||
update52EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
}else{
|
||||
println province+" "+city+" "+district+" "+street+" 没找到"
|
||||
}
|
||||
//修改100 community
|
||||
//修改89 community
|
||||
//修改52 community
|
||||
|
||||
}
|
||||
|
||||
|
||||
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
|
||||
|
||||
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictAliasProcess();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.fenQiProcess()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def fenQiProcess(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat","district")
|
||||
q.put("status", 0)
|
||||
q.put("master_district_name", 'master')
|
||||
def dbcursor = DbUtils.seaweed.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
|
||||
def lastObj = null
|
||||
for(def j : rets){
|
||||
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
|
||||
lastObj = j;
|
||||
break
|
||||
}
|
||||
}
|
||||
if(lastObj){
|
||||
println r
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("master_district_name","")
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-8
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
// task.addOldMatch()
|
||||
task.addOldMatch()
|
||||
// task.runDb()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
task.runMatch()
|
||||
// task.runMatch()
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
|
||||
if(!d['AJ'] && d['I']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
@@ -137,9 +137,10 @@ class GXDdistrictMatch {
|
||||
}
|
||||
|
||||
def runMatch(){
|
||||
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
|
||||
def city = "福州"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
def fileName = "D:\\数据\\福州\\建行测试匹配.xlsx"//11城重新匹配.xlsx 福州贝壳数据.xlsx
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['A']
|
||||
println d['C']
|
||||
if(d['B'] && d['A']){
|
||||
def region = RegionData.formatRegion(city, d['B'])
|
||||
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 名录集
|
||||
*/
|
||||
class MinglujiData {
|
||||
|
||||
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MinglujiData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def prov = "浙江省"
|
||||
def city = "嘉兴"
|
||||
// exportDatas(city)
|
||||
matchDev(prov, city)
|
||||
matchCommunity(city)
|
||||
|
||||
// tempMatchDev(prov)
|
||||
// saveMljData()
|
||||
|
||||
fiveLevelSplit(city)
|
||||
}
|
||||
def exportDatas(city){
|
||||
def datas = findMinglujiDatas(city)
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
|
||||
}
|
||||
def matchDev(prov, city){
|
||||
def devs = loadAllDevs(prov)
|
||||
|
||||
def i = 1;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if (d['D']) {
|
||||
def street = d['D']
|
||||
def splitStreet = ""
|
||||
if (street) {
|
||||
if (street.contains(d['C'])) {
|
||||
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
} else {
|
||||
splitStreet = street
|
||||
}
|
||||
}
|
||||
d['K'] = splitStreet
|
||||
}
|
||||
|
||||
if (d['I']) {
|
||||
def splitCommunity = ""
|
||||
if (d['C'] && d['I'].toString().contains(d['C'])) {
|
||||
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
|
||||
} else {
|
||||
splitCommunity = d['I']
|
||||
}
|
||||
d['L'] = splitCommunity
|
||||
}
|
||||
|
||||
|
||||
if (d['E'] && !d['N']) {
|
||||
def obj = findDevByName(d['E'])
|
||||
if (!obj) {
|
||||
obj = findDevByAlias(d['E'])
|
||||
if (!obj) {
|
||||
obj = findDebBySimlar(devs, d['E'], d['B'])
|
||||
if (!obj) {
|
||||
obj = findDevByPark(d['A'], d['B'], d['E'])
|
||||
if (!obj) {
|
||||
} else {
|
||||
d['N'] = "园区"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "相似"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "别名"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "相同"
|
||||
}
|
||||
if (obj) {
|
||||
d['M'] = obj.name
|
||||
d['O'] = obj.city
|
||||
d['P'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
i++
|
||||
println fileName +" 完成(匹配开发区不使用接口)"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def tempMatchDev(prov){
|
||||
def devs = loadAllDevs(prov)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
|
||||
if(d['U'] && !d['W'] ){
|
||||
def obj = findDevByName(d['U'])
|
||||
if(!obj){
|
||||
obj = findDevByAlias(d['U'])
|
||||
if(!obj){
|
||||
obj = findDebBySimlar(devs, d['U'], d['B'])
|
||||
if(!obj){
|
||||
obj = findDevByPark(d['A'], d['B'], d['U'])
|
||||
if (!obj) {
|
||||
}else{
|
||||
d['X'] = "园区"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "相似"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "别名"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "相同"
|
||||
}
|
||||
if(obj){
|
||||
d['W'] = obj.name
|
||||
d['Y'] = obj.city
|
||||
d['Z'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(city){
|
||||
def i = 1;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['A'] && d['B'] && d['L'] && !d['Q']){
|
||||
def list = findCommunityByName(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
|
||||
}else{
|
||||
d['T'] = "包含"
|
||||
}
|
||||
}else{
|
||||
d['T'] = "相同"
|
||||
}
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
def obj = list[0]
|
||||
d['Q'] = obj.name
|
||||
d['R'] = obj.street_name
|
||||
d['U'] = obj.area_name
|
||||
d['V'] = obj.city_name
|
||||
}else if(list.size()>=2){
|
||||
for(def l : list){
|
||||
if(d['C']==l.area_name){
|
||||
d['Q'] = l.name
|
||||
d['R'] = l.street_name
|
||||
d['U'] = l.area_name
|
||||
d['V'] = l.city_name
|
||||
break;
|
||||
}
|
||||
}
|
||||
d['S'] = list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
println fileName +" 运行完成"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fiveLevelSplit(city){
|
||||
def i = 1;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['J'] && !d['X']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['J'])
|
||||
if(jsonObject) {
|
||||
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
|
||||
d['X'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
def saveMljData(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("province", d['A'])
|
||||
obj.put("city", d['B'])
|
||||
obj.put("district", d['C'])
|
||||
obj.put("name", d['D'])
|
||||
obj.put("address", d['E'])
|
||||
obj.put("dev_name", d['F'])
|
||||
obj.put("committee", d['G'])
|
||||
obj.put("township", d['H'])
|
||||
company_100_CONN.save(obj)
|
||||
})
|
||||
}
|
||||
def loadAllDevs(prov){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
return development_zone_CONN.find(q).toList()
|
||||
}
|
||||
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias1 = null
|
||||
if(!d.alias){
|
||||
alias1 = []
|
||||
}else{
|
||||
alias1 = d.alias
|
||||
}
|
||||
if(!alias1.contains(d.name)){
|
||||
alias1 << d.name
|
||||
}
|
||||
if(alias1){
|
||||
try {
|
||||
for(def n : alias1){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}catch(exception){
|
||||
println alias1
|
||||
}
|
||||
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findCommunityByName(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", name)
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findCommunityBySimlar(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findMinglujiDatas(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("c_address",1)
|
||||
keys.put("c_city",1)
|
||||
keys.put("c_committee",1)
|
||||
keys.put("c_community",1)
|
||||
keys.put("c_company",1)
|
||||
keys.put("c_district",1)
|
||||
keys.put("c_street",1)
|
||||
keys.put("kind",1)
|
||||
keys.put("name",1)
|
||||
keys.put("province",1)
|
||||
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
|
||||
def rets = []
|
||||
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
|
||||
for(def d : datas){
|
||||
def street = d.c_street
|
||||
def splitStreet = ""
|
||||
if(street){
|
||||
if(street.contains(d.c_district)){
|
||||
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
}
|
||||
}
|
||||
def splitCommunity = ""
|
||||
if(d.c_committee){
|
||||
if(d.c_committee.contains(d.c_district)){
|
||||
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
|
||||
}
|
||||
}
|
||||
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+423
@@ -0,0 +1,423 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
|
||||
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
|
||||
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
// task.run();
|
||||
task.regionAndStreetProcess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = [""]
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
def regionAndStreetProcess(){
|
||||
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
|
||||
for(def city : citys){
|
||||
def datas = findAllPoisByCity(city)
|
||||
process(datas)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllPoisByCity(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findPois(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>=10000){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
// updateToEs2(datas)
|
||||
// updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
// updateToEs2(datas)
|
||||
// updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def process(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
dataRun(d)
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = running(d)
|
||||
rets << map
|
||||
if(map.update_status && map.update_status!=100){
|
||||
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
|
||||
}
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
updateToMG(map)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def regions = getFullRegion(d)
|
||||
def town = null
|
||||
def map =[:]
|
||||
if(d.location){
|
||||
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if(d.district){
|
||||
for(def rs : regions){
|
||||
if(rs.shape && rs.shape.coordinates){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(regionFlag){
|
||||
//判断township是否在统计局中
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, d.city, d.township)
|
||||
}
|
||||
if(town){
|
||||
if(town.street == d.township){
|
||||
//打标记
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = town.street
|
||||
}
|
||||
}else{
|
||||
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
|
||||
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
|
||||
if (mapTown) {
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = mapTown.street
|
||||
}else{
|
||||
if(d.district!=gaodeMap.district){
|
||||
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
//打标记“经纬度有误,不在区域围栏中”
|
||||
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
def dataRun(d){
|
||||
def regions = getFullRegion(d)
|
||||
if(d.location) {
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if (d.district) {
|
||||
for (def rs : regions) {
|
||||
if (rs.shape && rs.shape.coordinates) {
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (regionFlag) {//在围栏里
|
||||
//判断township是否在统计局中
|
||||
// if(d.township){
|
||||
// town = getSimilarTown(regions, d.city, d.township)
|
||||
// }
|
||||
|
||||
|
||||
}else{//不在转栏里,重新搜索
|
||||
|
||||
println(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.district)){
|
||||
names << r
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def getSimilarTown(regions, city, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(city+"#"+r.district)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.township='${township}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def updateToEs2(rows) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.update_status && map.update_status==2){
|
||||
map.remove("run_status")
|
||||
map.remove("update_status")
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def updateToMG(m){
|
||||
// def rets = []
|
||||
// for(def i=0;i<rows.size();i++){
|
||||
// def m = rows[i]
|
||||
def udata = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", m.id)
|
||||
if(udata){
|
||||
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println m.toString()+" 无法修改-无值"
|
||||
}
|
||||
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThardParthAnalyBatch extends BaseBatch{
|
||||
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThardParthAnalyBatch(), args)
|
||||
}
|
||||
|
||||
// static main(args) {
|
||||
// def task = new ThardParthAnalyBatch()
|
||||
// task.run()
|
||||
// }
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def fjwTotalCount = CommonDatautils.findAllCount("district")
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
println "统计月份:"+sDate+" 查询更新时间:"+mdate
|
||||
|
||||
def crawlTotalCount = findAllEsDataCount(mdate)
|
||||
println "抓取第三方小区总量:"+crawlTotalCount
|
||||
def synchTotalCount = findAllMongoDatasCount(sDate)
|
||||
println "同步到第三方库小区总量: "+synchTotalCount
|
||||
|
||||
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
|
||||
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
|
||||
|
||||
def synchDistCount = findAllMongoDistrictCount(sDate)
|
||||
println "同步到第三方库有价格小区量: "+synchDistCount
|
||||
|
||||
def matchTotalCount = findMatchSuccessCount(sDate)
|
||||
println "第三方匹配成功总量:"+matchTotalCount
|
||||
|
||||
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
|
||||
println "第三方匹配成功小区量:"+matchDistTotalCount
|
||||
|
||||
def precent = matchDistTotalCount*100.0/fjwTotalCount
|
||||
precent = DataHandleUtils.getDoubleRound(2, precent)
|
||||
println "匹配成功占比:"+precent+"%"
|
||||
|
||||
def noMatchTotalCount = findNoMatchCount(sDate)
|
||||
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
|
||||
|
||||
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findAllEsDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllEsDistDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllMongoDatasCount(s_date){
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findAllMongoDistrictCount(s_date){
|
||||
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("region",1)
|
||||
keys.put("name",1)
|
||||
def dbcursor = thirdPartyColl.find(q,keys)
|
||||
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findAllMongoDatasHavePricesCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessDistCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findNoMatchCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
|
||||
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_dist_total", fjwTotalCount)
|
||||
udata.put("crawl_total", crawlTotalCount)
|
||||
udata.put("synch_total", synchTotalCount)
|
||||
udata.put("synch_have_price_total", synchHavePriceTotalCount)
|
||||
udata.put("synch_have_price_district_total", synchDistCount)
|
||||
udata.put("match_total", matchTotalCount)
|
||||
udata.put("match_dist_total", matchDistTotalCount)
|
||||
udata.put("match_dist_rate", precent)
|
||||
udata.put("no_match_total", noMatchTotalCount)
|
||||
udata.put("c_date", new Date())
|
||||
if(obj){
|
||||
udata.put("m_date", new Date())
|
||||
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
udata.put("s_date", sDate)
|
||||
THIRD_PARTY_STAT.save(udata)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+366
-2
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
@@ -26,6 +30,9 @@ class TownMiddleProcess {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
@@ -61,6 +68,21 @@ class TownMiddleProcess {
|
||||
|
||||
// addDistrictDiffFlag()
|
||||
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
// addAliasFromExcel()
|
||||
|
||||
//增加简称及简拼
|
||||
// addSimpleName()
|
||||
|
||||
//根据民政部数据修改行政区划中间表
|
||||
// mzbDistrictUpate()
|
||||
|
||||
// addCity()
|
||||
}
|
||||
|
||||
|
||||
@@ -415,14 +437,16 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
@@ -800,4 +824,344 @@ class TownMiddleProcess {
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
});
|
||||
}
|
||||
|
||||
def updateCommunityDiffStreet(){
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city_name", d.city)
|
||||
query.put("area_name", d.district)
|
||||
query.put("street_name", d.street)
|
||||
def obj = COMMUNITY_CONN.findOne(query)
|
||||
if(!obj){
|
||||
query.put("street_name", d.district+d.street)
|
||||
obj = COMMUNITY_CONN.findOne(query)
|
||||
if(obj){
|
||||
def newStreet = obj.street_name.replace(d.district, "")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_name", newStreet)
|
||||
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
|
||||
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
|
||||
}else{
|
||||
println d.city +" "+d.district+" "+d.street
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updateTownPoline(){
|
||||
def city = "唐山市"
|
||||
def name = "高新技术开发区街道"
|
||||
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
|
||||
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("city", city).append("street",name)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def checkTown(){
|
||||
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def ad : dbcursor){
|
||||
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
|
||||
def count = TOWNPOLYLINE_COLL.count(polyline_q)
|
||||
if(count==1){
|
||||
println ad.city+" "+ ad.street+" "+count
|
||||
|
||||
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("polyline_source", "区域+街道匹配")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", ad._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateTownshipByRegion(){
|
||||
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("mzb_district_name",1)
|
||||
keys.put("mzb_district_code",1)
|
||||
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
|
||||
for(def d : dbcursor){
|
||||
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mzb_district_name",d.mzb_district_name)
|
||||
udata.put("mzb_district_code",d.mzb_district_code)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
def addAliasFromExcel(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
q.put("city", row['A'])
|
||||
q.put("street", row['D'])
|
||||
if(row['B']){
|
||||
q.put("district", row['B'])
|
||||
}
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
row['F'] = '是'
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(row['C'])){
|
||||
alias << row['C']
|
||||
}
|
||||
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", obj._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addSimpleName(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
|
||||
def prov = ProvinceData.getSimpleProvince(d.province)
|
||||
if(prov){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province_short", prov)
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", d._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
// def city = d.city.replace("市","").replace("区","").replace("县","")
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("city_short", city)
|
||||
// if(d.district && d.district.size()>=3){
|
||||
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
|
||||
// udata.put("district_short", region)
|
||||
// }
|
||||
// def updateQuery = new BasicDBObject("_id", d._id)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
// def city = RegionData.formatCity(d.city)
|
||||
// if(city && d.mzb_district_name){
|
||||
// def region = RegionData.formatRegion(city, d.mzb_district_name)
|
||||
// if(region){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("city_short", city)
|
||||
// udata.put("district_short", region)
|
||||
//
|
||||
// def updateQuery = new BasicDBObject("_id", d._id)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def mzbDistrictUpate(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
|
||||
if(!row['I']){
|
||||
def updateType = row['E']
|
||||
def prov = row['B']
|
||||
def newStreet = row['G']
|
||||
|
||||
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
|
||||
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['K'] = "存在"
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
}
|
||||
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
|
||||
if(row['F']){
|
||||
def dbo = null
|
||||
def code = row['F'].toString()
|
||||
if(!code.contains(" ")){
|
||||
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
|
||||
}
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
def alias = dbo.alias
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}
|
||||
if(row['D']){
|
||||
if(!alias.contains(row['D'])){
|
||||
row['K'] = "存在(加别名)"
|
||||
alias << row['D']
|
||||
addAlias(dbo._id, alias)
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}else if(updateType == "撤街道设街道"){
|
||||
row['K'] = "人工处理"
|
||||
}else if(updateType == "隶属关系变更"){
|
||||
row['K'] = "人工处理"
|
||||
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
|
||||
row['K'] = "人工处理"
|
||||
}else if(updateType =='撤销乡'){
|
||||
if(!row['F']){
|
||||
row['K'] = "人工处理"
|
||||
}
|
||||
}
|
||||
|
||||
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
|
||||
def dbo = findStreets(prov, newStreet, null)
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['K'] = "存在-编码不同"
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
def alias = dbo.alias
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}
|
||||
if(row['D']){
|
||||
if(!alias.contains(row['D'])){
|
||||
row['K'] = "存在(加别名)"
|
||||
alias << row['D']
|
||||
// addAlias(dbo._id, alias)
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addCity(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
|
||||
def citycode = null
|
||||
if(row['F']){
|
||||
citycode = row['F'].toString().substring(0, 4)
|
||||
}else if(row['C']){
|
||||
citycode = row['C'].toString().substring(0, 4)
|
||||
}
|
||||
|
||||
if(citycode){
|
||||
def q = new BasicDBObject()
|
||||
q.put("mzb_city_code", citycode+"00")
|
||||
q.put("type", "城市")
|
||||
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(dbo){
|
||||
row['O'] = dbo.city
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def findStreets(prov, newStreet, streetCode){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("province", new BasicDBObject('$regex', provPattern))
|
||||
q.put("street", newStreet)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def list = []
|
||||
def dbo = null
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
if(list && list.size()==1){
|
||||
dbo = list[0]
|
||||
}else if(list && list.size()>1){
|
||||
if(streetCode){
|
||||
for(def d : list){
|
||||
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
|
||||
dbo = d
|
||||
break;
|
||||
}else if(streetCode.contains(d.pid)){
|
||||
dbo = d
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return dbo
|
||||
}
|
||||
|
||||
def addAlias(id, alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("operator","高娟-批量")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
+632
@@ -0,0 +1,632 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
def fullCityMap = null
|
||||
def regionMap = null
|
||||
def commMap = null
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
// task.process()
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
}
|
||||
|
||||
|
||||
void process() {
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
|
||||
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
//
|
||||
// if(["中山","东莞"].contains(city)){
|
||||
// continue
|
||||
// }
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
// if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
// 相同且为统计局乡镇
|
||||
// 更正
|
||||
// 不同且不为统计局乡镇
|
||||
// 区域不同
|
||||
// 地图无乡镇街道
|
||||
// 城市不同
|
||||
|
||||
def stat(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
def rets = []
|
||||
rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
def map = [:]
|
||||
def datas = null
|
||||
if(file.exists()){
|
||||
datas = ExcelUtils.read(fileName,0, 1)
|
||||
for(def d : datas){
|
||||
def key = "空值"
|
||||
if(d['S']){
|
||||
key = d['S']
|
||||
}
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
}
|
||||
|
||||
println city
|
||||
map.each {k,v->
|
||||
println k+" "+v
|
||||
}
|
||||
println "--------------------------------------------------------------------------------------------"
|
||||
if(map){
|
||||
rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")]
|
||||
}
|
||||
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺")
|
||||
}
|
||||
|
||||
def updateTown(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
println fileName
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['S']!='相同且为统计局乡镇' && d['S']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['B'])
|
||||
q.put("name", d['C'])
|
||||
q.put("status", 0)
|
||||
q.put("cat","shop")
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", d['R'])
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkData(def dists){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << running(d)
|
||||
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return dists
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
|
||||
regions = []
|
||||
regions << fullCity.city
|
||||
}
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
def oldSimlarTown = null
|
||||
def newSimlarTown = null
|
||||
if(d.lng){
|
||||
d.lng = d.lng/10000000000.0
|
||||
d.lat = d.lat/10000000000.0
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, fullCity, d.township)
|
||||
}
|
||||
|
||||
if(town && town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
d.polygon_town = town.street
|
||||
}
|
||||
}
|
||||
d.strand_street = ""
|
||||
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
|
||||
if(mapTown){
|
||||
d.strand_street = mapTown.street
|
||||
if(mapTown.shape && mapTown.shape.coordinates){
|
||||
d.map_town_shape = "有"
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
|
||||
if(flag){
|
||||
d.map_polygon_town = mapTown.street
|
||||
}
|
||||
}
|
||||
}else{
|
||||
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
|
||||
if(oldSimlarTown){
|
||||
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
|
||||
d.old_similar_town_score = oldSimlarTown.score
|
||||
}
|
||||
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
|
||||
if(newSimlarTown){
|
||||
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
|
||||
d.new_similar_town_score = newSimlarTown.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def mapCity = null
|
||||
def mapRegion = null
|
||||
def mapTownship = null
|
||||
def mapAddr = null
|
||||
if(gaodeMap){
|
||||
if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
mapCity = gaodeMap.city
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}else{
|
||||
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
|
||||
mapRegion = mapCity
|
||||
}
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
}
|
||||
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
mapAddr = gaodeMap.format_addr
|
||||
}
|
||||
}
|
||||
d.mapCity = mapCity
|
||||
d.mapRegion = mapRegion
|
||||
d.mapTownship = mapTownship
|
||||
d.mapAddr = mapAddr
|
||||
if(mapCity && mapRegion){
|
||||
if(mapCity == fullCity.city){
|
||||
if(regions.contains(mapRegion)){
|
||||
if(mapTownship){
|
||||
if(town){
|
||||
d.old_tjj_township = "是"
|
||||
}else{
|
||||
d.old_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTown){
|
||||
d.new_tjj_township = "是"
|
||||
}else{
|
||||
d.new_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTownship == d.township){
|
||||
if(town){
|
||||
d.remark = "相同且为统计局乡镇"
|
||||
}else{
|
||||
d.remark = "非统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}else{
|
||||
if(mapTown){
|
||||
d.remark = "更正"
|
||||
}else{
|
||||
d.remark = "不同且不为统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d.remark = "地图无乡镇街道"
|
||||
d.opt = "地图无街道信息"
|
||||
}
|
||||
|
||||
}else{
|
||||
d.remark = "区域不同"
|
||||
d.opt = "经纬度可能跨区域"
|
||||
}
|
||||
}else{
|
||||
d.remark = "城市不同"
|
||||
d.opt = "经纬度可能跨城市"
|
||||
}
|
||||
}
|
||||
|
||||
if(d.map_town_shape && d.map_town_shape=="有"){
|
||||
if(!d.map_polygon_town){
|
||||
d.opt = "经纬度可疑"
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def getSimilarTownByStr(fullCity, townstr, townList){
|
||||
|
||||
def maxScore = 0
|
||||
def town = null
|
||||
for(def townObj : townList){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
|
||||
if(score > maxScore){
|
||||
maxScore = score
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
town.score = maxScore
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getSimilarTown(regions, fullCity, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(fullCity.city+"#"+r)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getFullRegion(fullCity, d){
|
||||
def regions = regionMap.get(fullCity.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
|
||||
def createDatas(dists){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
|
||||
for(def d : dists){
|
||||
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(township || district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateDiffRegionTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
|
||||
d['AA'] = lists.size()
|
||||
//只处理库中只有一个的
|
||||
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
|
||||
def newRegion = RegionData.formatRegion(d['B'], d['M'])
|
||||
if(newRegion){
|
||||
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
def dbo = lists[0]
|
||||
def arrs = d['X'].split(" ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", arrs[1])
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
|
||||
d['AB'] = "已处理"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def cleanTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['AC']){
|
||||
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
|
||||
if(dbo && dbo.township){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkTownFromMongo(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
citys = citys.reverse()
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
index++
|
||||
if (["中山", "东莞"].contains(city)) {
|
||||
continue
|
||||
}
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
|
||||
for (def d : dists) {
|
||||
if (d.township) {
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def town = getSimilarTown(regions, fullCity, d.township)
|
||||
if (!town) {
|
||||
town = findStreet(d.city, d.city, d.township)
|
||||
if (!town) {
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
|
||||
|
||||
println d.city + " " + d.region + " " + d.township + " " + d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println index + " " + city + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findStreet(city, district, street){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("city", new BasicDBObject('$regex', provPattern))
|
||||
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
return list[0]
|
||||
}else{
|
||||
println city+ " "+district+" "+street+" 有多个"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def updateTownMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['F']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mzb_district_name", d['F'])
|
||||
//
|
||||
// def updateQuery = new BasicDBObject()
|
||||
// updateQuery.put("city", d['C'])
|
||||
// updateQuery.put("district", d['D'])
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
|
||||
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
if(city){
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(region){
|
||||
def updateQuery = new BasicDBObject()
|
||||
updateQuery.put("category", "region")
|
||||
updateQuery.put("city", city)
|
||||
updateQuery.put("name", region)
|
||||
|
||||
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
|
||||
if(list && list.size()==1){
|
||||
def dbo = list[0]
|
||||
def query = new BasicDBObject()
|
||||
query.put("_id", dbo._id)
|
||||
|
||||
// def pattern = dbo.pattern
|
||||
// if(pattern.contains(d['F'])){
|
||||
// pattern.remove(d['F'])
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("org_name", d['F'])
|
||||
// udata.put("pattern",pattern)
|
||||
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
|
||||
}else{
|
||||
println city +" "+region+" "+list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDevMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("district", d['B'])
|
||||
q.put("name",d['C'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("township", d['D'])
|
||||
def list = dev_MAPPING_COLL_100.find(q).toList()
|
||||
if(list && list.size()==2){
|
||||
def obj = list[0]
|
||||
def removeQuery = new BasicDBObject()
|
||||
removeQuery.put("_id", obj._id)
|
||||
dev_MAPPING_COLL_100.remove(removeQuery)
|
||||
}else{
|
||||
println q
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateSpecCityTown(){
|
||||
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
|
||||
|
||||
// def citys = ["中山"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
|
||||
if(list.size()>1){
|
||||
println d['A'] +" "+d['C'] + " 有多个"
|
||||
}else if(list.size()==1){
|
||||
def dbo = list[0]
|
||||
def town = null
|
||||
// if(d['X']){
|
||||
// town = d['X']
|
||||
// }else
|
||||
if(d['R']){
|
||||
town = d['R']
|
||||
}
|
||||
if(!town && d['M']){
|
||||
town = d['M']
|
||||
}
|
||||
def region = RegionData.formatRegion(d['A'], town)
|
||||
if(region){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", region)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", town)
|
||||
udata.put("region", region)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -1,13 +1,46 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class TradeDatasDistBatch {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
// def
|
||||
def task = new TradeDatasDistBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
udata.put("name", d['C'])
|
||||
udata.put("alias", d['D'])
|
||||
alias_fang_COLL.save(udata)
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def flag = null
|
||||
// if(!dist){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }else{
|
||||
// flag = "名称匹配"
|
||||
// }
|
||||
// if(flag){
|
||||
// d['F'] = dist.city
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+17
-7
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
@@ -335,7 +336,7 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
@@ -352,18 +353,27 @@ class FangjiaClientUtils {
|
||||
public static def formatNinelevel(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
|
||||
def url = null
|
||||
address = address.replace("{","")
|
||||
int retryCount = 0
|
||||
while (retryCount < 3) {
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
if (content) {
|
||||
break
|
||||
}
|
||||
retryCount++
|
||||
}
|
||||
if (!content) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
// println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
println address +" "+content+" "+e.getMessage()
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
+5
-7
@@ -36,18 +36,16 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
|
||||
if(!rMap && address){
|
||||
if(!rMap && address && name){
|
||||
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true, name)
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false, null)
|
||||
}
|
||||
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||
// if (obj) {
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
|
||||
+12
-10
@@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -53,31 +53,33 @@ class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
def fangjiaDists = new HashSet()
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
|
||||
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
}
|
||||
|
||||
if(today > 10){
|
||||
// if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
|
||||
+25
-21
@@ -5,28 +5,19 @@ import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.*
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
@@ -49,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -59,7 +50,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
@@ -99,6 +90,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
println "txt Éú³É " + rows.size()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
@@ -130,7 +122,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def ccbPriceMap = null
|
||||
if(city == "上海"){
|
||||
ccbPriceMap = getCommunityPrices("上海")
|
||||
ccbPriceMap = [:]//getCommunityPrices("ÉϺ£")
|
||||
}
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
@@ -169,13 +161,25 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
|
||||
latch.await();
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getCommunityPrices(def city){
|
||||
def sql = "select commid, avg_price from community_base_price where year_month=${endDate} and cityname='${city}'";
|
||||
def map = [:]
|
||||
try{
|
||||
sqlInstance.eachRow(sql,{
|
||||
map.put(it.commid, it.avg_price)
|
||||
});
|
||||
}catch(Exception e){
|
||||
//getPpeGTE15Datas(city)
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city) {
|
||||
def q = new BasicDBObject()
|
||||
@@ -353,7 +357,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
def data = distCodeMap.get(trade.match_region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
@@ -512,13 +516,12 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
@@ -579,6 +582,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def getYouDaData(statMap){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
|
||||
+41
-11
@@ -67,13 +67,16 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
|
||||
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataForsaleBatch(), args);
|
||||
}
|
||||
@@ -86,15 +89,15 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19"
|
||||
def endDate = curMonth+"19" //正式运行时curMonth要改
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
def houseIdsfilter = null;//findAllHouseIdsByCity()
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
@@ -194,7 +197,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
@@ -270,7 +273,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
|
||||
|
||||
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
// if(rets1){
|
||||
// def newRets = []
|
||||
@@ -489,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
@@ -709,6 +712,33 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
def rets = []
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(3000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
@@ -889,11 +919,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
def findAllHouseIdsByCity(){
|
||||
def findAllHouseIdsByCity(cityid){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
// def houseIds = new HashSet()
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
|
||||
filter.put(it.gujia_sell_id)
|
||||
});
|
||||
|
||||
|
||||
+13
-4
@@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
@@ -48,19 +48,24 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
def rows = []
|
||||
|
||||
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
|
||||
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
|
||||
|
||||
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
|
||||
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
}
|
||||
println "【房价网内部】人工指导价总计:"+rows.size()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
@@ -118,11 +123,15 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(priceIdsfilter.mightContain(ps._id.toString())){
|
||||
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
|
||||
continue
|
||||
}
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
|
||||
+14
-1
@@ -1,7 +1,11 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
/**
|
||||
* 房价网内部使用
|
||||
*/
|
||||
@@ -9,7 +13,7 @@ class FjwPgsqlCommUtils {
|
||||
|
||||
public static def database = "fangjia"
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
public static def findAllDistrictIdsByOracle(def cityid){
|
||||
@@ -35,5 +39,14 @@ class FjwPgsqlCommUtils {
|
||||
sqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
public static def findGuiDingIdsByCity(){
|
||||
// ´´½¨²¼Â¡¹ýÂËÆ÷¶ÔÏó
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
|
||||
filter.put(it.gujia_fsid)
|
||||
});
|
||||
return filter
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+29
-151
@@ -35,11 +35,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -50,20 +47,23 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
if(sDate<202110){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
}
|
||||
month = month -1
|
||||
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
if(sDate == 201912){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
runDatas(citys)
|
||||
|
||||
//ͬ²½ÀúÊ·Êý¾Ý
|
||||
// def stop = false
|
||||
// def month = -1
|
||||
// while(!stop){
|
||||
// if(sDate>202404 && sDate<202407){
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
// println weekend.toString() +" "+sDate
|
||||
// runDatas(citys)
|
||||
// }
|
||||
// month = month -1
|
||||
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
// if(sDate == 201912){
|
||||
// stop = true;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
def runDatas(citys){
|
||||
@@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "·¿²ú³¬ÊÐ"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -389,139 +395,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData3(cityid){
|
||||
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def dataMap = [:]
|
||||
|
||||
dataMap.COMMID = r.COMMID
|
||||
dataMap.COMMNAME = r.COMMNAME
|
||||
dataMap.CITYID = r.CITYID
|
||||
dataMap.CITYNAME = r.CITYNAME
|
||||
dataMap.DISTID = r.DISTID
|
||||
dataMap.DISTNAME = r.DISTNAME
|
||||
dataMap.YEAR_MONTH = r.YEAR_MONTH
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(r.ZHUGE_AVG_PRICE){
|
||||
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ANJUKE_AVG_PRICE){
|
||||
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANG_AVG_PRICE){
|
||||
price_map.fang_avg_price = r.FANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.LIANJIA_AVG_PRICE){
|
||||
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.CITYHOUSE_AVG_PRICE){
|
||||
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.MAITIAN_AVG_PRICE){
|
||||
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.KE_AVG_PRICE){
|
||||
price_map.ke_avg_price = r.KE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TAOFANG_AVG_PRICE){
|
||||
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ZHONGYUAN_AVG_PRICE){
|
||||
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TBS_AVG_PRICE){
|
||||
price_map.tbs_avg_price = r.TBS_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.QFANG_AVG_PRICE){
|
||||
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGTAN_AVG_PRICE){
|
||||
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGDUODUO_AVG_PRICE){
|
||||
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.WIWJ_AVG_PRICE){
|
||||
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
|
||||
}
|
||||
dataMap.updatetime = r.UPDATETIME
|
||||
|
||||
allRows << dataMap
|
||||
});
|
||||
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
@@ -592,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+31
-23
@@ -33,10 +33,10 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
@@ -167,14 +167,14 @@ class AddPriceTest {
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['A']){
|
||||
// def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(map){
|
||||
// d['AK'] = map.avg_price
|
||||
d['P'] = map.calc_type
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
@@ -183,7 +183,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -205,7 +205,7 @@ class AddPriceTest {
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['X']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['X'] = thirdPrice
|
||||
@@ -220,8 +220,8 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
q.put("s_date", 202404)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202403))
|
||||
// q.put("s_date", 202404)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
@@ -241,12 +241,12 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['L']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['L'] = gxdPrice
|
||||
d['M'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -270,20 +270,28 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['N']){
|
||||
if(d['C'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
// if(!d['L']){
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['AJ'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
d['E'] = dist.name
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.name_cat
|
||||
// d['F'] = dist.estate_type2
|
||||
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
|
||||
// d['H'] = dist.master_district_name
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
// d['W'] = dbo.avg_price
|
||||
d['G'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
d['N'] = dbo.avg_price_2b
|
||||
// d['E'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+82
-1
@@ -27,13 +27,94 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
|
||||
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\对公地址(问题案例)跑一下结果.xlsx", { row ->
|
||||
if(row['B']){
|
||||
def parmas = [
|
||||
'address': row['B'],
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def str = getProxyResult(parmas);
|
||||
println str
|
||||
JSONObject result = JSONObject.fromObject(str)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['C'] = data.province
|
||||
}
|
||||
if(data.provinceCode && data.provinceCode!='null'){
|
||||
row['D'] = data.provinceCode
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['E'] = data.city
|
||||
}
|
||||
if(data.cityCode && data.cityCode!='null'){
|
||||
row['F'] = data.cityCode
|
||||
}
|
||||
|
||||
if(data.district && data.district!='null'){
|
||||
row['G'] = data.district
|
||||
}
|
||||
if(data.districtCode && data.districtCode!='null'){
|
||||
row['H'] = data.districtCode
|
||||
}
|
||||
|
||||
if(data.town && data.town!='null'){
|
||||
row['I'] = data.town
|
||||
}
|
||||
if(data.townCode && data.townCode!='null'){
|
||||
row['J'] = data.townCode
|
||||
}
|
||||
|
||||
if(data.village && data.village!='null'){
|
||||
row['K'] = data.village
|
||||
}
|
||||
if(data.villageCode && data.villageCode!='null'){
|
||||
row['L'] = data.villageCode
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['M'] = data.areaCode
|
||||
}
|
||||
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['N'] = "非农"
|
||||
}else{
|
||||
row['N'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['O'] = "否"
|
||||
}else{
|
||||
row['O'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['P'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['Q'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils89 {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
// connectionProperties.setProperty("user","developer");
|
||||
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
|
||||
connectionProperties.setProperty("user","elastic");
|
||||
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
if(results.size()%100000==0){
|
||||
println "."+ new Date()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+271
-62
File diff suppressed because one or more lines are too long
+1
-1
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def run(){
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
|
||||
+17
@@ -98,6 +98,23 @@ class InsidePolygon {
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<List<Point>> createPoint4(def pointList){
|
||||
List<List<Point>> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
List<Point> list = new ArrayList<>();
|
||||
for(def arr : pointArr){
|
||||
Point point = new Point();
|
||||
double x = arr[0];
|
||||
double y = arr[1];
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个经纬度点与区域的关系
|
||||
|
||||
+3
-3
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def districtMap = [:];
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
//查找出所有未进行匹配的数据
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
||||
// qdata.put("city","福州")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
// def esSources = ["搜房":"搜房"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDistrictPrice(), args);
|
||||
|
||||
+2
-1
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
|
||||
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.KeyGenerator;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
public class DESUtil {
|
||||
|
||||
private static final String ALGORITHM = "DES";
|
||||
|
||||
// 生成密钥
|
||||
public static SecretKey generateKey() throws Exception {
|
||||
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
|
||||
keyGenerator.init(56); // DES密钥长度为56位
|
||||
return keyGenerator.generateKey();
|
||||
}
|
||||
|
||||
// 将密钥转换为字符串
|
||||
public static String keyToString(SecretKey secretKey) {
|
||||
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
|
||||
}
|
||||
|
||||
// 将字符串转换为密钥
|
||||
public static SecretKey stringToKey(String keyStr) {
|
||||
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
|
||||
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
|
||||
}
|
||||
|
||||
// DES加密
|
||||
public static String encrypt(String data, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
|
||||
byte[] encryptedData = cipher.doFinal(data.getBytes());
|
||||
return Base64.getEncoder().encodeToString(encryptedData);
|
||||
}
|
||||
|
||||
// DES解密
|
||||
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey);
|
||||
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
|
||||
byte[] decryptedData = cipher.doFinal(decodedData);
|
||||
return new String(decryptedData);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
// 生成密钥
|
||||
// SecretKey secretKey = generateKey();
|
||||
// String keyStr = keyToString(secretKey);
|
||||
// System.out.println("密钥: " + keyStr);
|
||||
//
|
||||
//// // 加密数据
|
||||
// String data = "高娟";
|
||||
// String encryptedData = encrypt(data, secretKey);
|
||||
// System.out.println("加密后的数据: " + encryptedData);
|
||||
|
||||
// 解密数据
|
||||
SecretKey newSecretKey = stringToKey("tnb12345");
|
||||
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
|
||||
System.out.println("解密后的数据: " + decryptedData);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -186,6 +186,26 @@ public class DbUtils{
|
||||
return query
|
||||
}
|
||||
|
||||
public static BasicDBObject buildSeaweedQuery2(def city = null, def region = null, def name = null, def cat = "district", def status = 0){
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
if(city){
|
||||
query.append("city", city)
|
||||
}
|
||||
if(region){
|
||||
query.append("region", region)
|
||||
}
|
||||
if(name){
|
||||
query.append("name", name)
|
||||
}
|
||||
if(cat != null){
|
||||
query.append("cat", cat)
|
||||
}
|
||||
if(status != null){
|
||||
query.append("status", status)
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* 把小区修改成删除状态
|
||||
* @param id
|
||||
|
||||
+24
-1
@@ -76,7 +76,6 @@ class ElasticUtils {
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@@ -101,6 +100,30 @@ class ElasticUtils {
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
@@ -71,6 +71,10 @@ class MongoUtils {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
def key = db + name + domainUrl + port
|
||||
if (collectionMap.containsKey(key)) {
|
||||
return collectionMap[key]
|
||||
|
||||
@@ -156,6 +156,22 @@ public class ProvinceData {
|
||||
return PROVINCE_MAP.get(province);
|
||||
}
|
||||
|
||||
|
||||
public static String getSimpleProvince(String province){
|
||||
def prov = null
|
||||
def newDatas = data + GOVERNMENT_CITY_LIST
|
||||
for(def key : newDatas.keySet()){
|
||||
def list = newDatas.get(key)
|
||||
list << key
|
||||
if(list.contains(province)){
|
||||
prov = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
return prov;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取省份下的所有城市
|
||||
* @param province
|
||||
|
||||
+2
-1
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
|
||||
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
Reference in New Issue
Block a user