Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
majintao
2024-09-03 09:37:46 +08:00
970 changed files with 41507 additions and 12996 deletions
@@ -3,6 +3,7 @@ package com.fangjia.script.ana
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
@@ -24,7 +25,7 @@ class FindStageDistrictBatch extends BaseBatch {
@Override
void process(Object args) {
CommonUtils.findAllCity().each { def city ->
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
CommonDatautils.SEAWEED_COLL.find(DbUtils.buildSeaweedQuery2(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
if (dbo.name =~ STAGE_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
@@ -40,7 +41,6 @@ class FindStageDistrictBatch extends BaseBatch {
println(dbo)
}
}
}
}
}
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -72,19 +72,21 @@ for(def d : datas){
}
map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble())
map.area = d['F'].replace("平","").toString().toDouble()
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
if(d['I']){
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
}
map.trade_date = d['J']
// def trade_date = d['J']
// if(trade_date){
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
// }else{
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
// }
// else{
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
// }
map.source = "深房家族"
@@ -0,0 +1,40 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new AddThirdPriceSample()
task.run()
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
}
}
}, i)
println(i)
}
}
}
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def fangjiaDists = new HashSet()
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
def today = new Date().getDate()
println "starting 。。。。。。。。。"
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
for(def d : allDists){
fangjiaDists << d._id.toString()
}
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
println "加载建行小区量"
def ccbMaps = null
if(today<=10){
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
println "组装新数据"
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -35,7 +35,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
@@ -62,20 +62,20 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
def objectId = dateToObjectId(date)
println objectId
// def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
// def objectId = dateToObjectId(date)
// println objectId
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
//
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def citys =CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -189,7 +189,7 @@ class BigdataDealBatch extends BaseBatch{
})
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
latch.await();
return result
}
@@ -385,7 +385,7 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
def data = distCodeMap.get(trade.match_region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
@@ -720,22 +720,33 @@ class BigdataDealBatch extends BaseBatch{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(region || (!region && ["澄迈"].contains(t.city))) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
if(!rMap && t.address){
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
def flag = false
def address = null
if(t.address){
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
if(flag){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
address = t.address
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
if(!rMap && !t.name.toString().endsWith("小区")){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
}
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if (rMap && ["澄迈"].contains(rMap.city)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else{
if (rMap) {
println "区域不同"
}
@@ -752,7 +763,7 @@ class BigdataDealBatch extends BaseBatch{
}
}
);
)
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
@@ -7,18 +7,22 @@ import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.nio.charset.StandardCharsets
class CcbCommonUtils {
//"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照",
public static def citys_fjw_all = [ "武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
//"乌鲁木齐",
public static def citys_all = ["上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
@@ -233,7 +237,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -438,11 +442,14 @@ class CcbCommonUtils {
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
if(city){
q.put("city", city);
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","新泾三村")
// q.put("name","一米阳光一期")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
DBObject keys = new BasicDBObject();
if(fields){
@@ -452,9 +459,9 @@ class CcbCommonUtils {
}
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def rets = new HashSet()
def ids = new HashSet()
def ids = []
for(def d : dbsour){
if(kind){
if(d.address){
@@ -634,7 +641,8 @@ class CcbCommonUtils {
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("match_id", new BasicDBObject('$exists', false))
// q.put("district_name","中海寰宇天下")
// q.put("district_name","新明胡同")
// q.put("address","桂苑大道135号")
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
@@ -684,6 +692,17 @@ class CcbCommonUtils {
}
public static def findGuiDingIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
pgsqlInstance.eachRow("SELECT gujia_fsid FROM base.t0831_guided_price_gj_h",{it->
filter.put(it.gujia_fsid)
});
return filter
}
public static def findDistGovPrice(def city){
def ids = []
pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it->
@@ -720,6 +739,8 @@ class CcbCommonUtils {
if(count>0){
def udata = new BasicDBObject()
udata.put("trade_date", endDate)
udata.put("m_date", new Date())
udata.put("flag", "补漏变更成交时间")
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
}
}
@@ -389,7 +389,7 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
def citys = CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -47,16 +47,22 @@ class GuiDingPriceBatch extends BaseBatch{
def rows = []
CcbCommonUtils.deleteGuidedPrice(curDate)
def priceIdsfilter = CcbCommonUtils.findGuiDingIdsByCity()
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24")
// def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
//近3个月人工指导价
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
rows.addAll(rets)
println city + " " + rets.size()
}
@@ -80,7 +86,8 @@ class GuiDingPriceBatch extends BaseBatch{
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
println "人工指导价总计:"+rows.size()
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
saveToPgSql(rows)
//
@@ -142,11 +149,18 @@ class GuiDingPriceBatch extends BaseBatch{
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(priceIdsfilter.mightContain(ps._id.toString())){
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
continue
}
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
}
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
// if(d['G']){
// def obj = CommonDatautils.findDistrictById(d['G'])
// if(obj){
// d['H'] = obj.name
// }
// }
//})
////def url = "https://sh.anjuke.com"
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
@@ -602,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
////},0)
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
// if(d['A']){
// def dbo = CommonDatautils.findDistrictById(d['D'])
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def udata = new BasicDBObject()
//// udata.put("name_cat", 11)
@@ -620,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
// }
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
@@ -1289,35 +1331,49 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
// if(dist){
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// }
// }
if(!d['H'] && d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
d['H'] = dist.name
}
}
}
},0)
//
//// d['H'] = dist.name
// }
//// }
// }
//},0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
//ExcelUtils.multiThreadingModify(file, { d ->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['C'] = dist.city
// d['D'] = dist.region
// d['E'] = dist.name
// def random = RandomUtil.random(-300,50)
// d['F'] = random + d['B']
// }
// }
//})
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
//ExcelUtils.multiThreadingModify(filePath, { d ->
// if(d['B']){
@@ -1338,6 +1394,56 @@ ExcelUtils.multiThreadingModify(fileName, {d->
//
//})
//
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
//ExcelUtils.modify(filePath, { d ->
// if(d['A']){
//// def obj = CommonDatautils.findDistrictById(d['A'])
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
// if(list){
// if(list.size()==1){
// def dbo = list[0]
// if(dbo){
// def data = new BasicDBObject()
// data.put("region", d['B'])
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
// }
// }else{
// println d['C'] +" 有多个"
// }
// }
//// if(obj){
//// def lng = d['G']
//// def lat =d['H']
////
//// // 高德转百度
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
//// def lng2 = baidu.x
//// def lat2 = baidu.y
////
////
//// def data = new BasicDBObject()
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
//// data.put("lng", lng_new)
//// data.put("lat", lat_new)
////
//// Long dLat2 = lat2 * 10000000000L
//// Long dLng2 = lng2 * 10000000000L
//// data.put("lat2", dLat2)
//// data.put("lng2", dLng2)
////
////// data.put("status", 1)
//// data.put("m_date", new Date())
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
//// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
@@ -1439,6 +1545,9 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
return count
}
//citys = CcbCommonUtils.citys_all
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
@@ -0,0 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
if(d['G'] && d['G'] instanceof String){
def obj = CommonDatautils.findDistrictById(d['G'])
if(obj){
d['H'] = obj.name
}
}
})
@@ -47,15 +47,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
// def gocPrices = CcbCommonUtils.findDistGovPrice("深圳")
// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?"
// pgsqlInstance.withBatch(500, sql){ ps->
// for(def id : gocPrices){
// ps.addBatch(id)
// }
// }
//
def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
@@ -265,6 +257,12 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
if(third.get("csjj_esf_price")){
price_map.csjj_avg_price = third.get("csjj_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -311,6 +309,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "房产超市"
}else if(k=="csjj_avg_price"){
web = "城市居家"
}
ret << sourceMap.get(web)
@@ -528,6 +530,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -592,6 +595,14 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -599,8 +610,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
if(value && value.size()>1){ //csjj_esf_price 国信达成交
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price && !value.csjj_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
@@ -32,6 +32,10 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -40,8 +44,10 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -499,8 +505,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2023-08-01"
def endDate = "2023-10-01"
def startDate = "2024-04-01"
def endDate = "2024-06-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -584,6 +590,36 @@ class TradeStatLog {
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def exportTrade2(){
// def startDate = "2024-06-01"
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
if(d['S'] && !d['W']){
def dist = CommonDatautils.findDistrictById(d['S'])
if(!dist.name_cat && dist.name_cat!=0){
d['W'] = "name_cat过滤"
}
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
d['W'] = "小区过滤"
}
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
def map = findDealByCommId(d['B'], d['S'])
def count = map.get(key)
if(count>=2){
d['W'] = "重复"
}
}
println d['A']
})
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
@@ -637,11 +673,39 @@ class TradeStatLog {
return trades
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
ids << r.gujia_bds_mdl_id
}
})
return ids
}
def findDealByCommId(city, commid){
def q = new BasicDBObject()
q.put("match_id", commid)
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
def dbcursor = DEAL_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
return map
}
def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
def size = regionDatas.size()
final CountDownLatch latch = new CountDownLatch(size)
def totalCount = new AtomicLong()
for(def i = 0;i<regionDatas.size();i++){
final def arr = regionDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
totalCount.addAndGet(1)
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
if(row){
rets << row
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// };
// })
}
latch.await()
// latch.await()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
def allDistRegionPreMap = [:]
def allLastMonthMap = [:]
def allLastYearMap = [:]
def allCcbPriceMap = [:]
def allCcbPriceCaseCountMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
allDistricts.addAll(districts)
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
allPriceMap = allPriceMap + priceMap
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
// allPriceMap = allPriceMap + priceMap
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
allRentMap = allRentMap + rentMap
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceMap = allCcbPriceMap + datasMap
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
def cityRankMap = findCommunityCityRank(datas)
allCityRankMap = allCityRankMap + cityRankMap
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
allLastYearMap = allLastYearMap + lastYearMonthMap
}
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(cityLists){
rets.addAll(cityLists)
}
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
final CountDownLatch latch = new CountDownLatch(distCount)
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(row){
rets << row
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def sellDataType = 1
def rentDataType = 1
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
//房源查询是否按建行均价筛选房源
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
def sell_avg_price = null
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def sell_count = 0
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
if(allCcbPriceMap.containsKey(dist._id.toString())){
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
}
}
if(sell_count){
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
}
}
}
if(sellMap){
sell_count = sellMap.avg_price.size()
if(sell_count>3){
sell_avg_price = avgValue(sellMap.avg_price)
if(sellMap){
sell_avg_area = avgValue(sellMap.area)
sell_avg_total = avgValue(sellMap.total_price)
}else{
sell_count = 0
sellDataType = null
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// if(sell_count>3){
// if(allCcbPriceMap.containsKey(dist._id.toString())){
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
// }else{
// sell_count = 0
// sellDataType = null
// }
// }else{
// sell_count = 0
// sellDataType = null
// }
}
}
def rent_avg_price = null
def rent_avg_area = null
def rent_avg_total = null
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
rentDataType = null
}
}
if(!sell_avg_price){
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
// if(!sell_avg_price){
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
// }
if(!rent_avg_price){
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def avgPrice = null
if(region){
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
}else{
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
}
if (sell_count < 10) {
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
sellStartDate = startDate
}
if (sell_count) {
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
}
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
return datas;
}
def findCommunityPriceByCityMap(cityCode, month){
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
})
return dataMap;
}
def findCommunityPriceCountByCityMap(cityCode, month){
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
if(r.gujiabdsrt3mlgcasenum){
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
}
})
return dataMap;
}
def findCcbCityPrice(cityCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def avgPrice = null
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
return regionPriceMap;
}
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
def price = null
pgsqlInstance.eachRow(sql, { r ->
price = r.gujia_bds_clc_avprc
})
return price;
}
def districtCityPrictRate(cityPrice, datas){
def distCityRateMap = [:]
@@ -146,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
@@ -221,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
@@ -253,7 +253,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
}
@@ -295,7 +295,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
}
@@ -327,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
@@ -356,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
@@ -384,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
@@ -422,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
@@ -452,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
@@ -485,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
@@ -517,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
@@ -552,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
@@ -576,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
@@ -676,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -716,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -987,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -998,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
@@ -1072,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
@@ -1084,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportThirdPartyPrice2(price_date, new_dists) {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1132,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1215,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
@@ -1261,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
@@ -1304,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1319,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1393,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1460,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1501,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1537,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1570,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1609,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1652,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1689,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1737,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1784,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1845,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
@@ -1856,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataForsale(publishDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1913,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataForsale(publishDate,endDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -2098,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
@@ -13,11 +13,13 @@ import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.poi.ss.util.RegionUtil
import org.bson.Document
import org.bson.types.ObjectId
@@ -117,6 +119,8 @@ class CommonDatautils {
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -274,7 +278,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","泰瑞家园")
// q.put("name","托斯卡纳庄园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -283,7 +287,7 @@ class CommonDatautils {
}
}
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -1205,6 +1209,13 @@ class CommonDatautils {
return rets;
}
public static def findAllCount(String cat){
DBObject q = new BasicDBObject();
q.put("cat", cat);
q.put("status", 0)
return SEAWEED_MONGO_COLL.count(q)
}
public static def findDistrictsByName(String city, String name, String cat){
@@ -1843,39 +1854,6 @@ class CommonDatautils {
}
return priceMap;
}
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
if(cat == "district"){
BasicDBObject queryObj = new BasicDBObject();
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
queryCondition.put("estate_type", queryObj);
}else if(cat == "office"){
queryCondition.put("estate_type", "写字楼");
}else if(cat == "shop"){
queryCondition.put("estate_type", "商铺");
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public static def queryFirstDistrictPrice(city, region, name){
@@ -1914,6 +1892,39 @@ class CommonDatautils {
}
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
if(cat == "district"){
BasicDBObject queryObj = new BasicDBObject();
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
queryCondition.put("estate_type", queryObj);
}else if(cat == "office"){
queryCondition.put("estate_type", "写字楼");
}else if(cat == "shop"){
queryCondition.put("estate_type", "商铺");
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public static def queryDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -4571,6 +4582,28 @@ class CommonDatautils {
return townMap
}
static def findAllTownRanges(){
def q = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("town", 1)
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
def townMap = [:]
for(def d : dbcursor){
def lists = townMap.get(d.city)
if(!lists){
lists = []
}
def t = [:]
t.region = d.region
t.town = d.town
lists << t
townMap.put(d.city, lists)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
@@ -4605,6 +4638,148 @@ class CommonDatautils {
}
static def queryFullNameByCity(){
def q = new BasicDBObject()
q.put("type","城市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def cityMap = [:]
for(def d : dbcursor){
def city = RegionData.formatCity(d.city)
if(city){
cityMap.put(city, d)
}else{
//println d.city +" 格式化不出"
}
}
return cityMap
}
static def queryDistNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
def regions = regionMap.get(d.city)
if(!regions){
regions = []
}
regions << d
regionMap.put(d.city, regions)
}else{
println d.province+" "+d.district +" 格式化不出民政部区域"
}
}
println "区域加载完成"
return regionMap
}
static def queryCommunityByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "乡镇街道 加载完成"
return townMap
}
static def queryTownByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city
def list = townMap.get(key)
if(!list){
list = []
}
def t = [:]
t.region = d.district
t.town = d.street
list << t
townMap.put(key, list)
}
return townMap
}
static def queryDistNameCity(){
def q = new BasicDBObject()
@@ -4632,6 +4807,21 @@ class CommonDatautils {
}
static def queryDistNameCityByMapping(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.district_short && d.city_short){
retMap.put(d.city_short+"#"+d.district_short, d)
}
}
return retMap;
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
@@ -4675,7 +4865,93 @@ class CommonDatautils {
}
static def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
def road = ""
def name = ""
def type = ""
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
if(addressComponent.streetNumber){
if(addressComponent.streetNumber.street){
road = addressComponent.streetNumber.street
}
if(addressComponent.streetNumber.number){
road = road + addressComponent.streetNumber.number
}
}
if(addressComponent.building){
if(addressComponent.building.name){
name = addressComponent.building.name
}
if(addressComponent.building.type){
type = addressComponent.building.type
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
map.road = road
map.type = type
map.name = name
return map;
}
static def saveOptLog(city, region, name, cat, opt, user_name){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category",cat);
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", opt)
obj.put("user_name", user_name)
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -264,12 +264,12 @@ class DistrictSellStatBatch extends BaseBatch {
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
def priceMap = [:]
for(def d : dbcursor){
if(city == "¸£ÖÝ"){
if(d.avg_price_2b && d.value_type!=1){
d.avg_price = d.avg_price_2b
if(!priceMap.containsKey(d.region +"##"+d.district_name+"##"+d.estate_type)){
if(city == "¸£ÖÝ"){
if(d.avg_price_2b && d.value_type!=1){
d.avg_price = d.avg_price_2b
}
}
}
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d)
}
}
@@ -0,0 +1,79 @@
package com.fangjia.script.gj
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class GreenBuild {
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new GreenBuild()
task.run()
}
def run(){
def datas = findAll()
def rets = match(datas)
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
}
def match(datas){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
data.match_id = resultMap.get("id").toString()
data.match_city = resultMap.get("city")
data.match_district = resultMap.get("district")
data.match_name = resultMap.get("name")
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
}else{
rets << [data._id.toString(), data.city, data.district, data.name]
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await();
return rets
}
def findAll(){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
}
}
@@ -13,12 +13,18 @@ import net.sf.json.groovy.JsonSlurper
import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
}
if(suggestName){
url = url +"&suggestName="+suggestName
}
if(suggestDistrict){
url = url +"&suggestDistrict="+suggestDistrict
}
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
@@ -12,8 +12,8 @@ class OpenUserBatch {
def task = new OpenUserBatch();
// task.run();
// task.test();
// task.addShenZhenFjtUser();
task.addFujianFjtUser()
task.addShenZhenFjtUser();
// task.addFujianFjtUser()
}
def test(){
@@ -46,20 +46,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252608 + i;
def id = 252893 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
onLineSqlInstance.execute(sql3)
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.jsoup.Jsoup
class BaiduBaikeTown {
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
static main(args) {
def task = new BaiduBaikeTown()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
if(!row['E'] && row['A']=="郑州经济技术开发区"){
def q = new BasicDBObject()
q.put("desc",new BasicDBObject('$ne', 'null'))
q.put("name",row['A'])
def d = dzb_COLL.findOne(q)
def townMap = parse(d.desc, d.name)
row['D'] = townMap.town_str1
row['F'] = townMap.town_str2
// def ret = []
// ret << ["名称", "基本信息", "目录", "全文"]
// for(def d : dbcursor){
// def townMap = parse(d.desc, d.name)
// if(townMap.town){
// println d.name+" "+townMap.town
// }
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
// }
}
})
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
}
def parse(desc, name){
def map = [:]
if(desc){
def doc = Jsoup.parse(desc)
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
def s = doc.toString()
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
def jsonObj = JSONObject.fromObject(dataStr)
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
def towns = []
if(jsonObj && jsonObj.catalog){
def catalogs = jsonObj.catalog
def index = null
for(def log : catalogs){
def curIndex = log.index
if(log.title=="行政区划"){
index = log.index
}
if(index && curIndex.toString().startsWith(index+"-")){
towns << log.title
}
}
}
map.town_str1 = str
if(town){
map.town = town
}else{
map.town = ""
}
if(towns){
map.towns = towns.join(",")
}else{
map.towns = ""
}
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
map.town_str2 = str2
return map
}
map.town_str1 = ""
map.town_str2 = ""
map.town = ""
map.towns = ""
return map
}
}
@@ -3,8 +3,11 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -15,21 +18,28 @@ import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import java.util.regex.Pattern
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
static main(args) {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
// task.devSplitRoad()
task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -38,17 +48,33 @@ class DevProcess {
// task.updateCityAndDist()
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址
task.matchDevAndAddAddress()
// task.addDevAddres3()
// task.matchDevAndAddAddress()
// task.stat()
// task.addNewDev()
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo43()
// task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
// task.addDevShape()//添加开发区围栏
// task.matchDevTownship()//添加乡镇街道
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
// task.matchNewCrawlDev()
// println DateUtils.add("2024-7-24", 7);
// task.addKfc()
// task.devToPark() //开发区下辖园区
// task.checkTown()
// task.repeatSave()
}
@@ -166,69 +192,177 @@ class DevProcess {
}
}
def devSplitRoad(){
def i = 0;
while(i<1){
def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\浙江-湖州.xlsx"
def findDev( name){
def obj = findDevByName(name)
if(!obj){
obj = findDevByAlias(name)
if(obj){
obj.match_way = "别名"
}
ExcelUtils.multiThreadingModify(fileName,{d->
/*if(d['C'] && !d['V']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['V'] = jsonObject.get("province")
}
}else{
obj.match_way = "相同"
}
return obj
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['W'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['X'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['Y'] = jsonObject.get("street")
if(streetNumber!='null'){
d['Z'] = jsonObject.get("streetNumber")
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias1 = null
if(!d.alias){
alias1 = []
}else{
alias1 = d.alias
}
if(!alias1.contains(d.name)){
alias1 << d.name
}
if(alias1){
try {
for(def n : alias1){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
}*/
if(d['C'] && !d['J']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(jsonObject){
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['J'] = jsonObject.get("province")
}
}catch(exception){
println alias1
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['K'] = jsonObject.get("city")
}
}
if(last){
break
}
}
return last
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['L'] = jsonObject.get("district")
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
if(jsonObject && street && streetNumber && street!="null"){
d['M'] = jsonObject.get("street")
if(streetNumber!='null'){
d['N'] = jsonObject.get("streetNumber")
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def devSplitRoad(){
def i = 1;
while(i<20){
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['C'] && !d['F']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(jsonObject1) {
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
def devName = jsonObject1.get("dev")
def obj = findDev(devName)
d['F'] = devName
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
d['S'] = jsonObject1.detail.dev_confi
}
if (obj) {
d['O'] = obj._id.toString()
d['P'] = obj.name
d['U'] = obj.match_way
}
}
}
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
d['O'] = jsonObject.get("dev")
}
}
}
},0)
if(d['E'] && !d['M']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
d['G'] = jsonObject.get("province")
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
d['H'] = jsonObject.get("city")
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
d['I'] = jsonObject.get("district")
}
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
d['J'] = jsonObject.get("township")
}
if (jsonObject && street && streetNumber && street != "null") {
d['K'] = jsonObject.get("street")
def road = jsonObject.get("street")
if (streetNumber != 'null') {
d['L'] = jsonObject.get("streetNumber")
road = road + jsonObject.get("streetNumber")
} else {
d['N'] = "信息不完整"
}
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
d['N'] = "原地址不含路"
}
if (!d['N']) {
def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$')
if (!flag) {
d['N'] = "路非法"
}
def roadsize = jsonObject.get("street").size()
if (!d['N'] && (roadsize < 3 || roadsize > 7)) {
d['N'] = "路太长|路太短"
}
}
}else{
d['N'] = "无地址"
}
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
def devName = jsonObject.get("dev")
def obj = findDev(devName)
d['M'] = devName
if (obj) {
d['Q'] = obj._id.toString()
d['R'] = obj.name
d['V'] = obj.match_way
}
if(jsonObject.containsKey("detail") && jsonObject.detail.dev_confi!='null'){
d['T'] = jsonObject.detail.dev_confi
}
}
}
},0)
}
i++
println fileName+" 完成"
}
}
@@ -295,14 +429,27 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
if(!d['P'] && d['R']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
if(!d['I'] && d['J']){
def q = new BasicDBObject()
q.put("name", d['R'])
// q.put("name", d['J'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['J']]))
def name = d['J'].replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['P'] = obj._id.toString()
d['Q'] = obj.name
d['I'] = "库中有开发区-相似2"
d['K'] = obj.name
// d['C'] = obj._id.toString()
// d['D'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
// }else{
// d['E'] = "包含"
// }
// d['E'] = "别名"
// d['F'] = obj.name
}
}
},0);
@@ -469,6 +616,38 @@ class DevProcess {
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def addDevAddres3(){
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
if(d['A']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
d['C'] = obj._id.toString()
def q2 = new BasicDBObject()
q2.put("name", d['A'])
q2.put("address", d['B'])
q2.put("type","地址")
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(!sameObj){
index++
def udata = new BasicDBObject()
udata.put("province", obj.province)
udata.put("city", obj.city)
udata.put("district", obj.district)
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address", d['B'])
DEV_MAPPING_CONN.save(udata)
}
}
}
},0)
println index
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def stat(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
def total = 0
@@ -593,27 +772,27 @@ class DevProcess {
});
}
def synchDevTo43(){
def dbcursor = development_zone_CONN.find()
def synchDevTo100(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
def rets = []
for(def r : dbcursor){
rets << r
}
DEVELOPMENT_ZONE_43_COLL.insert(rets)
development_zone_CONN.insert(rets)
}
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
if(d['R']){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
if(d['E']){
def q = new BasicDBObject()
q.put("name", d['R'])
q.put("name", d['E'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['P'] = obj._id.toString()
d['Q'] = obj.name
def address = d['L']+d['M']
d['F'] = obj._id.toString()
d['G'] = obj.name
def address = d['D']
q.put("address", address)
// def street = d['G']
// q.put("township", street)
@@ -631,6 +810,7 @@ class DevProcess {
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address",address)
udata.put("c_date", new Date())
// udata.put("type", "乡镇街道")
// udata.put("township", street)
DEV_MAPPING_CONN.save(udata)
@@ -641,6 +821,44 @@ class DevProcess {
},0);
}
def matchDevTownship(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
if(d['D']){
def q = new BasicDBObject()
q.put("name", d['D'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['F'] = obj._id.toString()
def street = d['E']
q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){
def udata = new BasicDBObject()
udata.put("province", obj.province)
if(obj.city){
udata.put("city", obj.city)
}
if(obj.district){
udata.put("district", obj.district)
}
udata.put("name",obj.name)
udata.put("type", "乡镇街道")
udata.put("township", street)
// udata.put("type", "区域")
// udata.put("dev_district", street)
udata.put("c_date",new Date())
DEV_MAPPING_CONN.save(udata)
println obj.name +" "+street
}
}
}
},1);
}
def addAlias(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
def q = new BasicDBObject()
@@ -665,4 +883,184 @@ class DevProcess {
}
})
}
def addDevShape(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d->
if(d['W']){
def s = d['S']
JSONObject jsonObject = JSONObject.fromObject(s);
Map<String, Object> map = DataHandleUtils.JsonToMap(jsonObject);
def udata = new BasicDBObject()
udata.put("shape", map)
development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata))
}
})
}
def addDevStreet(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
if(d['J']=="是"){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道"))
for(def obj : dbcursor){
def o = new BasicDBObject()
o.put("province" , obj.province)
o.put("city" , obj.city)
o.put("district" , obj.mzb_district_name)
o.put("name" , d['E'])
o.put("type" , "乡镇街道")
o.put("township" , obj.street)
DEV_MAPPING_CONN.save(o)
}
}
})
}
def matchNewCrawlDev(){
def query = new BasicDBObject()
query.put("match_id", new BasicDBObject('$exists', false))
def list = kfq_COLL.find(query).toList()
for(def d : list){
def q = new BasicDBObject()
// q.put("name", d.name)
q.put('alias', new BasicDBObject('$in',[d.name]))
def obj = development_zone_CONN.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("match_id", obj._id.toString())
udata.put("match_name", obj.name)
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addKfc(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
if(d['I']){
if(d['I'] == "添加"){
def udata = new BasicDBObject()
udata.put("create_time" , new Date())
udata.put("industry",d['D'])
if(d['E'].contains("国家级")){
udata.put("level" ,"国家级")
}else if(d['E'].contains("省级")){
udata.put("level" ,"省级")
}
udata.put("name" ,d['F'])
udata.put("province" ,d['G'])
udata.put("type" ,d['H'])
udata.put("city" ,d['B'])
udata.put("district" ,d['J'])
development_zone_CONN.save(udata)
}else{
def q = new BasicDBObject()
q.put("name", d['I'])
def obj = development_zone_CONN.findOne(q)
if(obj){
def alias = []
if(obj.alias){
alias = obj.alias
}
if(!alias.contains(d['F'])){
alias << d['F']
}
if(alias && alias.size()!=obj.alias.size()){
def updateData = new BasicDBObject()
updateData.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
}
}
}
}
})
}
def devToPark(){
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
def rets = []
rets << ["开发区名称", "下辖街道", "城市", "区域"]
for(def d : datas){
if(d['E']){
def arrs = d['E'].split("、")
def q = new BasicDBObject().append("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
for(def park : arrs){
rets << [d['A'], park, obj.city, obj.district]
}
}
}
}
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
}
def checkTown(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
if(!d['E']){
if(d['A']){
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
// if(jsonObject1) {
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
// def township = jsonObject1.get("township")
// def district = jsonObject1.get("district")
// d['E'] = "接口"
// if(district && district!='null'){
// d['F'] = district
// }
// if(township && township!='null'){
// d['G'] = township
// }
//
// d['H'] = jsonObject1.get("townCode")
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
// d['I'] = jsonObject1.get("provinceCode")
// }
// }
// }
def q = new BasicDBObject()
q.put("city", d['C'])
// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
q.put("type", "乡镇街道")
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
// q.put("street", d['B'])
// q.put('alias', new BasicDBObject('$in',[d['B']]))
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
d['E'] = "可入库-忽略区域模糊2"
d['F'] = obj.district
d['G'] = obj.street
}
}
}
},0)
}
def repeatSave(){
def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject())
def list = []
for(def map : dbcursor){
def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"]
def udata = new BasicDBObject()
map.each{k,v->
if(columns.contains(k)){
if(v instanceof String){
if(k=="_id"){
udata.put("_id", new ObjectId(v))
}else{
udata.put(k, v)
}
}
}
}
list << udata
}
DEV_MAPPING_CONN2.insert(list)
}
}
@@ -0,0 +1,281 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
// task.addDevAddress(path)
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
o.put("city", d['C'])
o.put("district", d['D'])
o.put("name", d['A'])
o.put("level", d['E'])
o.put("type", d['F'])
if(d['G']){
o.put("township", d['G'])
}
if(d['H']){
def arr = d['H'].split(",")
if(arr){
o.put("alias", arr)
}
}
def id = addMongoDB(o)
o.remove("_id")
o.id = id.toString()
o.remove("create_time")
addEs(o, "192.168.0.89", "development_zone")
addEs(o, "192.168.0.52", "development_zone")
d['I'] = "已完成"
}else{
println d['A'] +" 已存在"
}
}
})
}
def addMongoDB(o){
o.put("create_time", new Date())
development_zone_100_CONN.save(o)
DEVELOPMENT_ZONE_43_COLL.save(o)
return o._id
}
def addEs(map, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
q.put("township", d['C'])
}else if(d['B'] == "地址"){
q.put("address", d['C'])
}else if(d['B'] == "园区"){
q.put("park", d['C'])
}
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
if(!mappingObj){
def o = new BasicDBObject()
o.put("province", obj.province)
o.put("city", obj.city)
o.put("district", obj.district)
o.put("name", obj.name)
o.put("type", d['B'])
if(d['B'] == "乡镇街道"){
o.put("township", d['C'])
}else if(d['B'] == "地址"){
o.put("address", d['C'])
}else if(d['B'] == "园区"){
o.put("park", d['C'])
}
def date = new Date()
o.put("c_date", date)
o.put("version", date.getTime())
DEV_MAPPING_100_CONN.save(o)
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
d['D'] = "已完成"
}else{
d['D'] = "重复"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
d['D'] = "不存在"
println d['A'] +" 不存在"
}
}
},1)
}
def updateHasGov(province, city, district, street, hasGov){
def q = new BasicDBObject()
q.put("province", province)
q.put("city", city)
q.put("district", district)
q.put("street", street)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
//修改43
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
//修改89 administrative_mapping
update89EsAdministrativeMapping(province, city, district, street, hasGov)
//修改89 administrative_mapping
update52EsAdministrativeMapping(province, city, district, street, hasGov)
}else{
println province+" "+city+" "+district+" "+street+" 没找到"
}
//修改100 community
//修改89 community
//修改52 community
}
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
}
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
task.run()
// task.run()
task.fenQiProcess()
}
def run(){
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
}
})
}
def fenQiProcess(){
def q = new BasicDBObject()
q.put("cat","district")
q.put("status", 0)
q.put("master_district_name", 'master')
def dbcursor = DbUtils.seaweed.find(q)
def rets = []
for(def d : dbcursor){
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
}
for(def r : rets){
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def lastObj = null
for(def j : rets){
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
lastObj = j;
break
}
}
if(lastObj){
println r
def udata = new BasicDBObject()
udata.put("master_district_name","")
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
// task.addOldMatch()
task.addOldMatch()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
task.runMatch()
// task.runMatch()
}
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
@@ -137,9 +137,10 @@ class GXDdistrictMatch {
}
def runMatch(){
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
def city = "福州"
ExcelUtils.modify(fileName, { d ->
def fileName = "D:\\数据\\福州\\建行测试匹配.xlsx"//11城重新匹配.xlsx 福州贝壳数据.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['A']
println d['C']
if(d['B'] && d['A']){
def region = RegionData.formatRegion(city, d['B'])
@@ -0,0 +1,372 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* 名录集
*/
class MinglujiData {
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
static main(args) {
def task = new MinglujiData()
task.run()
}
def run(){
def prov = "浙江省"
def city = "嘉兴"
// exportDatas(city)
matchDev(prov, city)
matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
fiveLevelSplit(city)
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
}
def matchDev(prov, city){
def devs = loadAllDevs(prov)
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName, { d ->
if (d['D']) {
def street = d['D']
def splitStreet = ""
if (street) {
if (street.contains(d['C'])) {
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
} else {
splitStreet = street
}
}
d['K'] = splitStreet
}
if (d['I']) {
def splitCommunity = ""
if (d['C'] && d['I'].toString().contains(d['C'])) {
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
} else {
splitCommunity = d['I']
}
d['L'] = splitCommunity
}
if (d['E'] && !d['N']) {
def obj = findDevByName(d['E'])
if (!obj) {
obj = findDevByAlias(d['E'])
if (!obj) {
obj = findDebBySimlar(devs, d['E'], d['B'])
if (!obj) {
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
} else {
d['N'] = "园区"
}
} else {
d['N'] = "相似"
}
} else {
d['N'] = "别名"
}
} else {
d['N'] = "相同"
}
if (obj) {
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
}
}
})
}
i++
println fileName +" 完成(匹配开发区不使用接口)"
}
}
def tempMatchDev(prov){
def devs = loadAllDevs(prov)
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
if(d['U'] && !d['W'] ){
def obj = findDevByName(d['U'])
if(!obj){
obj = findDevByAlias(d['U'])
if(!obj){
obj = findDebBySimlar(devs, d['U'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['U'])
if (!obj) {
}else{
d['X'] = "园区"
}
}else{
d['X'] = "相似"
}
}else{
d['X'] = "别名"
}
}else{
d['X'] = "相同"
}
if(obj){
d['W'] = obj.name
d['Y'] = obj.city
d['Z'] = obj.district
}
}
})
}
def matchCommunity(city){
def i = 1;
while(i<20){
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['A'] && d['B'] && d['L'] && !d['Q']){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
d['U'] = obj.area_name
d['V'] = obj.city_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
d['U'] = l.area_name
d['V'] = l.city_name
break;
}
}
d['S'] = list.size()
}
}
}
}, 0)
println fileName +" 运行完成"
}
i++
}
}
def fiveLevelSplit(city){
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
}
File file = new File(fileName);
if (file.exists()) {
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['J'] && !d['X']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['J'])
if(jsonObject) {
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
d['X'] = jsonObject.get("address")
}
}
}
})
}
i++
}
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
obj.put("district", d['C'])
obj.put("name", d['D'])
obj.put("address", d['E'])
obj.put("dev_name", d['F'])
obj.put("committee", d['G'])
obj.put("township", d['H'])
company_100_CONN.save(obj)
})
}
def loadAllDevs(prov){
def q = new BasicDBObject()
q.put("province", prov)
return development_zone_CONN.find(q).toList()
}
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias1 = null
if(!d.alias){
alias1 = []
}else{
alias1 = d.alias
}
if(!alias1.contains(d.name)){
alias1 << d.name
}
if(alias1){
try {
for(def n : alias1){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
}catch(exception){
println alias1
}
}
if(last){
break
}
}
return last
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def findCommunityByName(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", name)
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findCommunityBySimlar(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findMinglujiDatas(city){
def keys = new BasicDBObject()
keys.put("c_address",1)
keys.put("c_city",1)
keys.put("c_committee",1)
keys.put("c_community",1)
keys.put("c_company",1)
keys.put("c_district",1)
keys.put("c_street",1)
keys.put("kind",1)
keys.put("name",1)
keys.put("province",1)
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
def rets = []
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
for(def d : datas){
def street = d.c_street
def splitStreet = ""
if(street){
if(street.contains(d.c_district)){
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
}
}
def splitCommunity = ""
if(d.c_committee){
if(d.c_committee.contains(d.c_district)){
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
}
}
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
}
return rets
}
}
@@ -0,0 +1,423 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
static main(args) {
def task = new PoiLocation();
// task.run();
task.regionAndStreetProcess()
}
def run(){
def citys = [""]
citys = citys.reverse()
for(def city : citys){
findPois(city)
}
}
def regionAndStreetProcess(){
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
for(def city : citys){
def datas = findAllPoisByCity(city)
process(datas)
}
}
def findAllPoisByCity(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
def index = 0
for(def d : dbcursor){
rets << d
if(rets.size()>=10000){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
// updateToEs2(datas)
// updateToMG(datas)
rets = []
print "."
}
}
if(rets.size()>0){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
// updateToEs2(datas)
// updateToMG(datas)
rets = []
print "."
}
return rets;
}
def process(datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
dataRun(d)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
rets << map
if(map.update_status && map.update_status!=100){
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
}
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
updateToMG(map)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def running(d){
def regions = getFullRegion(d)
def town = null
def map =[:]
if(d.location){
//判断区域否在围栏里
def regionFlag = null
if(d.district){
for(def rs : regions){
if(rs.shape && rs.shape.coordinates){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if(regionFlag){
//判断township是否在统计局中
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
if(town){
if(town.street == d.township){
//打标记
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = town.street
}
}else{
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
if (mapTown) {
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = mapTown.street
}else{
if(d.district!=gaodeMap.district){
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
}
}
}else{
//打标记“经纬度有误,不在区域围栏中”
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
return map;
}
def dataRun(d){
def regions = getFullRegion(d)
if(d.location) {
//判断区域否在围栏里
def regionFlag = null
if (d.district) {
for (def rs : regions) {
if (rs.shape && rs.shape.coordinates) {
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if (regionFlag) {//在围栏里
//判断township是否在统计局中
// if(d.township){
// town = getSimilarTown(regions, d.city, d.township)
// }
}else{//不在转栏里,重新搜索
println(d)
}
}
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.district)){
names << r
}
}
}
return names;
}
def getSimilarTown(regions, city, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.update_status && map.update_status==2){
map.remove("run_status")
map.remove("update_status")
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def updateToMG(m){
// def rets = []
// for(def i=0;i<rows.size();i++){
// def m = rows[i]
def udata = new BasicDBObject()
m.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
def q = new BasicDBObject()
q.put("_id", m.id)
if(udata){
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
}else{
println m.toString()+" 无法修改-无值"
}
// }
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
}
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
}
@@ -0,0 +1,159 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThardParthAnalyBatch(), args)
}
// static main(args) {
// def task = new ThardParthAnalyBatch()
// task.run()
// }
@Override
void process(Object args) {
run()
}
def run(){
def fjwTotalCount = CommonDatautils.findAllCount("district")
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
println "统计月份:"+sDate+" 查询更新时间:"+mdate
def crawlTotalCount = findAllEsDataCount(mdate)
println "抓取第三方小区总量:"+crawlTotalCount
def synchTotalCount = findAllMongoDatasCount(sDate)
println "同步到第三方库小区总量: "+synchTotalCount
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
def synchDistCount = findAllMongoDistrictCount(sDate)
println "同步到第三方库有价格小区量: "+synchDistCount
def matchTotalCount = findMatchSuccessCount(sDate)
println "第三方匹配成功总量:"+matchTotalCount
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
println "第三方匹配成功小区量:"+matchDistTotalCount
def precent = matchDistTotalCount*100.0/fjwTotalCount
precent = DataHandleUtils.getDoubleRound(2, precent)
println "匹配成功占比:"+precent+"%"
def noMatchTotalCount = findNoMatchCount(sDate)
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
}
def findAllEsDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllEsDistDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllMongoDatasCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
return thirdPartyColl.count(q)
}
def findAllMongoDistrictCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
q.put("avg_price", new BasicDBObject('$gt', 0))
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("region",1)
keys.put("name",1)
def dbcursor = thirdPartyColl.find(q,keys)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findAllMongoDatasHavePricesCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
return thirdPartyColl.count(q)
}
def findMatchSuccessCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
return thirdPartyColl.count(q)
}
def findMatchSuccessDistCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
def dbcursor = thirdPartyColl.find(q)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findNoMatchCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
return thirdPartyColl.count(q)
}
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
def udata = new BasicDBObject()
udata.put("fjw_dist_total", fjwTotalCount)
udata.put("crawl_total", crawlTotalCount)
udata.put("synch_total", synchTotalCount)
udata.put("synch_have_price_total", synchHavePriceTotalCount)
udata.put("synch_have_price_district_total", synchDistCount)
udata.put("match_total", matchTotalCount)
udata.put("match_dist_total", matchDistTotalCount)
udata.put("match_dist_rate", precent)
udata.put("no_match_total", noMatchTotalCount)
udata.put("c_date", new Date())
if(obj){
udata.put("m_date", new Date())
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
}else{
udata.put("s_date", sDate)
THIRD_PARTY_STAT.save(udata)
}
}
}
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -26,6 +30,9 @@ class TownMiddleProcess {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
@@ -61,6 +68,21 @@ class TownMiddleProcess {
// addDistrictDiffFlag()
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
// addAliasFromExcel()
//增加简称及简拼
// addSimpleName()
//根据民政部数据修改行政区划中间表
// mzbDistrictUpate()
// addCity()
}
@@ -415,14 +437,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -800,4 +824,344 @@ class TownMiddleProcess {
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
});
}
def updateCommunityDiffStreet(){
def q = new BasicDBObject("type", "乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
def query = new BasicDBObject()
query.put("city_name", d.city)
query.put("area_name", d.district)
query.put("street_name", d.street)
def obj = COMMUNITY_CONN.findOne(query)
if(!obj){
query.put("street_name", d.district+d.street)
obj = COMMUNITY_CONN.findOne(query)
if(obj){
def newStreet = obj.street_name.replace(d.district, "")
def udata = new BasicDBObject()
udata.put("street_name", newStreet)
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
}else{
println d.city +" "+d.district+" "+d.street
}
}
}
}
def updateTownPoline(){
def city = "唐山市"
def name = "高新技术开发区街道"
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("city", city).append("street",name)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
def checkTown(){
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def ad : dbcursor){
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
def count = TOWNPOLYLINE_COLL.count(polyline_q)
if(count==1){
println ad.city+" "+ ad.street+" "+count
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("polyline_source", "区域+街道匹配")
def updateQuery = new BasicDBObject("_id", ad._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
def updateTownshipByRegion(){
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
def keys = new BasicDBObject()
keys.put("mzb_district_name",1)
keys.put("mzb_district_code",1)
keys.put("city",1)
keys.put("district",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
for(def d : dbcursor){
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
def udata = new BasicDBObject()
udata.put("mzb_district_name",d.mzb_district_name)
udata.put("mzb_district_code",d.mzb_district_code)
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
def addAliasFromExcel(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
def q = new BasicDBObject("type", "乡镇街道")
q.put("city", row['A'])
q.put("street", row['D'])
if(row['B']){
q.put("district", row['B'])
}
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
row['F'] = '是'
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(row['C'])){
alias << row['C']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
def updateQuery = new BasicDBObject("_id", obj._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
})
}
def addSimpleName(){
def q = new BasicDBObject()
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
def prov = ProvinceData.getSimpleProvince(d.province)
if(prov){
def udata = new BasicDBObject()
udata.put("province_short", prov)
def updateQuery = new BasicDBObject("_id", d._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
// def city = d.city.replace("市","").replace("区","").replace("县","")
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// if(d.district && d.district.size()>=3){
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
// udata.put("district_short", region)
// }
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// def city = RegionData.formatCity(d.city)
// if(city && d.mzb_district_name){
// def region = RegionData.formatRegion(city, d.mzb_district_name)
// if(region){
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// udata.put("district_short", region)
//
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// }
// }
}
}
def mzbDistrictUpate(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
if(!row['I']){
def updateType = row['E']
def prov = row['B']
def newStreet = row['G']
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
}
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
if(row['F']){
def dbo = null
def code = row['F'].toString()
if(!code.contains(" ")){
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
}
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}else if(updateType == "撤街道设街道"){
row['K'] = "人工处理"
}else if(updateType == "隶属关系变更"){
row['K'] = "人工处理"
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
row['K'] = "人工处理"
}else if(updateType =='撤销乡'){
if(!row['F']){
row['K'] = "人工处理"
}
}
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
def dbo = findStreets(prov, newStreet, null)
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在-编码不同"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
// addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}
})
}
def addCity(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
def citycode = null
if(row['F']){
citycode = row['F'].toString().substring(0, 4)
}else if(row['C']){
citycode = row['C'].toString().substring(0, 4)
}
if(citycode){
def q = new BasicDBObject()
q.put("mzb_city_code", citycode+"00")
q.put("type", "城市")
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(dbo){
row['O'] = dbo.city
}
}
})
}
def findStreets(prov, newStreet, streetCode){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
q.put("province", new BasicDBObject('$regex', provPattern))
q.put("street", newStreet)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def list = []
def dbo = null
for(def d : dbcursor){
list << d
}
if(list && list.size()==1){
dbo = list[0]
}else if(list && list.size()>1){
if(streetCode){
for(def d : list){
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
dbo = d
break;
}else if(streetCode.contains(d.pid)){
dbo = d
break;
}
}
}
}
return dbo
}
def addAlias(id, alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("operator","高娟-批量")
def updateQuery = new BasicDBObject("_id", id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
@@ -0,0 +1,632 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TownshipStrandUpdate {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def fullCityMap = null
def regionMap = null
def commMap = null
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// task.updateSpecCityTown()
}
void process() {
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
citys = citys.reverse()
for(def city : citys){
//
// if(["中山","东莞"].contains(city)){
// continue
// }
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
// if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
// }
}
}
// 相同且为统计局乡镇
// 更正
// 不同且不为统计局乡镇
// 区域不同
// 地图无乡镇街道
// 城市不同
def stat(){
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
def rets = []
rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
def map = [:]
def datas = null
if(file.exists()){
datas = ExcelUtils.read(fileName,0, 1)
for(def d : datas){
def key = "空值"
if(d['S']){
key = d['S']
}
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
}
println city
map.each {k,v->
println k+" "+v
}
println "--------------------------------------------------------------------------------------------"
if(map){
rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")]
}
}
ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺")
}
def updateTown(){
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(file.exists()){
println fileName
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['S']!='相同且为统计局乡镇' && d['S']){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['B'])
q.put("name", d['C'])
q.put("status", 0)
q.put("cat","shop")
def udata = new BasicDBObject()
udata.put("township", d['R'])
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
})
}
}
}
def checkData(def dists){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets << running(d)
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return dists
}
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
regions = []
regions << fullCity.city
}
def gaodeMap = null
def town = null
def mapTown = null
def oldSimlarTown = null
def newSimlarTown = null
if(d.lng){
d.lng = d.lng/10000000000.0
d.lat = d.lat/10000000000.0
if(d.township){
town = getSimilarTown(regions, fullCity, d.township)
}
if(town && town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
d.polygon_town = town.street
}
}
d.strand_street = ""
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
if(mapTown){
d.strand_street = mapTown.street
if(mapTown.shape && mapTown.shape.coordinates){
d.map_town_shape = "有"
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
if(flag){
d.map_polygon_town = mapTown.street
}
}
}else{
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
if(oldSimlarTown){
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
d.old_similar_town_score = oldSimlarTown.score
}
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
if(newSimlarTown){
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
d.new_similar_town_score = newSimlarTown.score
}
}
}
}
def mapCity = null
def mapRegion = null
def mapTownship = null
def mapAddr = null
if(gaodeMap){
if(gaodeMap.city && gaodeMap.city!="[]"){
mapCity = gaodeMap.city
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}else{
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
mapRegion = mapCity
}
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
}
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
mapAddr = gaodeMap.format_addr
}
}
d.mapCity = mapCity
d.mapRegion = mapRegion
d.mapTownship = mapTownship
d.mapAddr = mapAddr
if(mapCity && mapRegion){
if(mapCity == fullCity.city){
if(regions.contains(mapRegion)){
if(mapTownship){
if(town){
d.old_tjj_township = "是"
}else{
d.old_tjj_township = "否"
}
if(mapTown){
d.new_tjj_township = "是"
}else{
d.new_tjj_township = "否"
}
if(mapTownship == d.township){
if(town){
d.remark = "相同且为统计局乡镇"
}else{
d.remark = "非统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}else{
if(mapTown){
d.remark = "更正"
}else{
d.remark = "不同且不为统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}
}else{
d.remark = "地图无乡镇街道"
d.opt = "地图无街道信息"
}
}else{
d.remark = "区域不同"
d.opt = "经纬度可能跨区域"
}
}else{
d.remark = "城市不同"
d.opt = "经纬度可能跨城市"
}
}
if(d.map_town_shape && d.map_town_shape=="有"){
if(!d.map_polygon_town){
d.opt = "经纬度可疑"
}
}
return d;
}
def getSimilarTownByStr(fullCity, townstr, townList){
def maxScore = 0
def town = null
for(def townObj : townList){
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
if(score > maxScore){
maxScore = score
town = townObj
}
}
if(town){
town.score = maxScore
}
return town
}
def getSimilarTown(regions, fullCity, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(fullCity.city+"#"+r)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def getFullRegion(fullCity, d){
def regions = regionMap.get(fullCity.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
names << r.district
}
}
}
return names;
}
def createDatas(dists){
def rets = []
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
for(def d : dists){
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
}
return rets;
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(township || district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
d['AB'] = "已处理"
}
}
}
})
}
def cleanTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['AC']){
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dbo && dbo.township){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
}
}
})
}
def checkTownFromMongo(){
def citys = CommonDatautils.findAllCity()
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
citys = citys.reverse()
def index = 0
for(def city : citys) {
index++
if (["中山", "东莞"].contains(city)) {
continue
}
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
for (def d : dists) {
if (d.township) {
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def town = getSimilarTown(regions, fullCity, d.township)
if (!town) {
town = findStreet(d.city, d.city, d.township)
if (!town) {
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
println d.city + " " + d.region + " " + d.township + " " + d.name
}
}
}
}
println index + " " + city + " 完成"
}
}
def findStreet(city, district, street){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
q.put("city", new BasicDBObject('$regex', provPattern))
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
if(list){
if(list.size()==1){
return list[0]
}else{
println city+ " "+district+" "+street+" 有多个"
}
}
return null
}
def updateTownMapping(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['F']){
// def udata = new BasicDBObject()
// udata.put("mzb_district_name", d['F'])
//
// def updateQuery = new BasicDBObject()
// updateQuery.put("city", d['C'])
// updateQuery.put("district", d['D'])
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
def city = RegionData.formatCity(d['C'])
if(city){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def updateQuery = new BasicDBObject()
updateQuery.put("category", "region")
updateQuery.put("city", city)
updateQuery.put("name", region)
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
if(list && list.size()==1){
def dbo = list[0]
def query = new BasicDBObject()
query.put("_id", dbo._id)
// def pattern = dbo.pattern
// if(pattern.contains(d['F'])){
// pattern.remove(d['F'])
// }
def udata = new BasicDBObject()
udata.put("org_name", d['F'])
// udata.put("pattern",pattern)
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
}else{
println city +" "+region+" "+list.size()
}
}
}
}
})
}
def updateDevMapping(){
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("district", d['B'])
q.put("name",d['C'])
q.put("type", "乡镇街道")
q.put("township", d['D'])
def list = dev_MAPPING_COLL_100.find(q).toList()
if(list && list.size()==2){
def obj = list[0]
def removeQuery = new BasicDBObject()
removeQuery.put("_id", obj._id)
dev_MAPPING_COLL_100.remove(removeQuery)
}else{
println q
}
})
}
def updateSpecCityTown(){
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
// def citys = ["中山"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
ExcelUtils.modify(fileName,{d->
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
if(list.size()>1){
println d['A'] +" "+d['C'] + " 有多个"
}else if(list.size()==1){
def dbo = list[0]
def town = null
// if(d['X']){
// town = d['X']
// }else
if(d['R']){
town = d['R']
}
if(!town && d['M']){
town = d['M']
}
def region = RegionData.formatRegion(d['A'], town)
if(region){
def q = new BasicDBObject()
q.put("region", region)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", town)
udata.put("region", region)
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
}else{
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
}
}
})
}
}
}
@@ -1,13 +1,46 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class TradeDatasDistBatch {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
static main(args) {
// def
def task = new TradeDatasDistBatch()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("alias", d['D'])
alias_fang_COLL.save(udata)
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def flag = null
// if(!dist){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }else{
// flag = "名称匹配"
// }
// if(flag){
// d['F'] = dist.city
// d['G'] = dist.region
// d['H'] = dist.name
// }
})
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -335,7 +336,7 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -352,18 +353,27 @@ class FangjiaClientUtils {
public static def formatNinelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
def url = null
address = address.replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
if (content) {
break
}
retryCount++
}
if (!content) {
return null
}
try {
jsonObj = JSONObject.fromObject(content)
// println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
println address +" "+content+" "+e.getMessage()
}
return jsonObj;
}
@@ -36,18 +36,16 @@ class SeaweedDataMatchProcess {
}
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(!rMap && address){
if(!rMap && address && name){
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
rMap = OpenApiUtils.matchDistrict(city, address, region, true, name)
}
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false, null)
}
// if(rMap && rMap.id && region.contains(rMap.region)) {
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
// if (obj) {
@@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -53,31 +53,33 @@ class FjwBigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
def fangjiaDists = new HashSet()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
for(def d : dists){
fangjiaDists << d._id.toString()
}
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
}
if(today > 10){
// if(today > 10){
delPgData()
saveToPgSql(rows)
}
// }
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -5,28 +5,19 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.*
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
@@ -49,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -59,7 +50,7 @@ class FjwBigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
@@ -99,6 +90,7 @@ class FjwBigdataDealBatch extends BaseBatch{
}
println "txt Éú³É " + rows.size()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
@@ -130,7 +122,7 @@ class FjwBigdataDealBatch extends BaseBatch{
def ccbPriceMap = null
if(city == "上海"){
ccbPriceMap = getCommunityPrices("上海")
ccbPriceMap = [:]//getCommunityPrices("ÉϺ£")
}
for(def i=0;i<rets.size();i++){
@@ -169,13 +161,25 @@ class FjwBigdataDealBatch extends BaseBatch{
})
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
latch.await();
return result
}
def getCommunityPrices(def city){
def sql = "select commid, avg_price from community_base_price where year_month=${endDate} and cityname='${city}'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.avg_price)
});
}catch(Exception e){
//getPpeGTE15Datas(city)
}
return map
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
@@ -353,7 +357,7 @@ class FjwBigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
def data = distCodeMap.get(trade.match_region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
@@ -512,13 +516,12 @@ class FjwBigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
@@ -579,6 +582,7 @@ class FjwBigdataDealBatch extends BaseBatch{
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
@@ -67,13 +67,16 @@ class FjwBigdataForsaleBatch extends BaseBatch{
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
runBatch(new FjwBigdataForsaleBatch(), args);
}
@@ -86,15 +89,15 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19"
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
def houseIdsfilter = null;//findAllHouseIdsByCity()
for(def city : citys){
index++
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
@@ -194,7 +197,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
} catch (Exception e) {
e.printStackTrace()
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
} finally {
latch.countDown()
}
@@ -270,7 +273,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
// def newRets = []
@@ -489,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -709,6 +712,33 @@ class FjwBigdataForsaleBatch extends BaseBatch{
return rets;
}
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
int retryCount = 0;
boolean success = false;
def rets = []
while (retryCount < MAX_TRY_COUNT && !success) {
try {
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
// 这里可以选择在重试之前等待一段时间
Thread.sleep(3000); // 例如等待1秒
}
}
if (!success) {
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
}
return rets;
}
/**
* 是否是别墅
* @return
@@ -889,11 +919,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(){
def findAllHouseIdsByCity(cityid){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
filter.put(it.gujia_sell_id)
});
@@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
@@ -48,19 +48,24 @@ class FjwGuiDingPriceBatch extends BaseBatch{
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
rows.addAll(rets)
println city + " " + rets.size()
}
}
println "【房价网内部】人工指导价总计:"+rows.size()
saveToPgSql(rows)
}
@@ -118,11 +123,15 @@ class FjwGuiDingPriceBatch extends BaseBatch{
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(priceIdsfilter.mightContain(ps._id.toString())){
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
continue
}
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
@@ -1,7 +1,11 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import groovy.sql.Sql
import java.nio.charset.StandardCharsets
/**
* 房价网内部使用
*/
@@ -9,7 +13,7 @@ class FjwPgsqlCommUtils {
public static def database = "fangjia"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
@@ -35,5 +39,14 @@ class FjwPgsqlCommUtils {
sqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// ´´½¨²¼Â¡¹ýÂËÆ÷ÔÏó
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
filter.put(it.gujia_fsid)
});
return filter
}
}
@@ -35,11 +35,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -50,20 +47,23 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def stop = false
def month = -1
while(!stop){
if(sDate<202110){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
}
month = month -1
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
if(sDate == 201912){
stop = true;
}
}
runDatas(citys)
//ͬ²½ÀúÊ·Êý¾Ý
// def stop = false
// def month = -1
// while(!stop){
// if(sDate>202404 && sDate<202407){
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
// println weekend.toString() +" "+sDate
// runDatas(citys)
// }
// month = month -1
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
// if(sDate == 201912){
// stop = true;
// }
// }
}
def runDatas(citys){
@@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "·¿²ú³¬ÊÐ"
}
ret << sourceMap.get(web)
@@ -389,139 +395,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
@@ -592,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -33,10 +33,10 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addThirdPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
@@ -167,14 +167,14 @@ class AddPriceTest {
}
def addCcbPrice(filepath){
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['A']){
// def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
// d['AK'] = map.avg_price
d['P'] = map.calc_type
d['F'] = map.avg_price
// d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
@@ -183,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -205,7 +205,7 @@ class AddPriceTest {
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['X']) {
def dist = CommonDatautils.findDistrictById(d['Y'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['X'] = thirdPrice
@@ -220,8 +220,8 @@ class AddPriceTest {
q.put("city", city)
q.put("region", region)
q.put("name", name)
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
q.put("s_date", 202404)
q.put("s_date", new BasicDBObject('$gte', 202403))
// q.put("s_date", 202404)
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
@@ -241,12 +241,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['L']) {
def dist = CommonDatautils.findDistrictById(d['E'])
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['L'] = gxdPrice
d['M'] = gxdPrice
}
}
})
@@ -270,20 +270,28 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['N']){
if(d['C'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['C'])
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['AJ'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
// d['H'] = dist.master_district_name
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
// d['W'] = dbo.avg_price
d['G'] = dbo.avg_price
// d['X'] = dbo.cal_type
d['N'] = dbo.avg_price_2b
// d['E'] = dbo.avg_price_2b
}
}
}
}
// }
// }
})
}
@@ -27,13 +27,94 @@ class QiliangTest {
// task.run()
def data = [
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
}
def run(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\对公地址(问题案例)跑一下结果.xlsx", { row ->
if(row['B']){
def parmas = [
'address': row['B'],
'type': '1' //1:对公 0:对私
]
def str = getProxyResult(parmas);
println str
JSONObject result = JSONObject.fromObject(str)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['C'] = data.province
}
if(data.provinceCode && data.provinceCode!='null'){
row['D'] = data.provinceCode
}
if(data.city && data.city!='null'){
row['E'] = data.city
}
if(data.cityCode && data.cityCode!='null'){
row['F'] = data.cityCode
}
if(data.district && data.district!='null'){
row['G'] = data.district
}
if(data.districtCode && data.districtCode!='null'){
row['H'] = data.districtCode
}
if(data.town && data.town!='null'){
row['I'] = data.town
}
if(data.townCode && data.townCode!='null'){
row['J'] = data.townCode
}
if(data.village && data.village!='null'){
row['K'] = data.village
}
if(data.villageCode && data.villageCode!='null'){
row['L'] = data.villageCode
}
if(data.areaCode && data.areaCode!='null'){
row['M'] = data.areaCode
}
if(data.farmer == "0"){
row['N'] = "非农"
}else{
row['N'] = "农"
}
if(data.isGovt == "0"){
row['O'] = "否"
}else{
row['O'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['P'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['Q'] = data.govtAddr?data.govtAddr:''
}
}
}
}
})
}
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.xiamen
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils89 {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
// connectionProperties.setProperty("user","developer");
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
connectionProperties.setProperty("user","elastic");
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
}
})
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
File diff suppressed because one or more lines are too long
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
@@ -98,6 +98,23 @@ class InsidePolygon {
}
return list;
}
public static List<List<Point>> createPoint4(def pointList){
List<List<Point>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point> list = new ArrayList<>();
for(def arr : pointArr){
Point point = new Point();
double x = arr[0];
double y = arr[1];
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
//查找出所有未进行匹配的数据
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -0,0 +1,69 @@
package com.fangjia.script.util
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
public class DESUtil {
private static final String ALGORITHM = "DES";
// 生成密钥
public static SecretKey generateKey() throws Exception {
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
keyGenerator.init(56); // DES密钥长度为56位
return keyGenerator.generateKey();
}
// 将密钥转换为字符串
public static String keyToString(SecretKey secretKey) {
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
}
// 将字符串转换为密钥
public static SecretKey stringToKey(String keyStr) {
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
}
// DES加密
public static String encrypt(String data, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
byte[] encryptedData = cipher.doFinal(data.getBytes());
return Base64.getEncoder().encodeToString(encryptedData);
}
// DES解密
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.DECRYPT_MODE, secretKey);
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
byte[] decryptedData = cipher.doFinal(decodedData);
return new String(decryptedData);
}
public static void main(String[] args) {
try {
// 生成密钥
// SecretKey secretKey = generateKey();
// String keyStr = keyToString(secretKey);
// System.out.println("密钥: " + keyStr);
//
//// // 加密数据
// String data = "高娟";
// String encryptedData = encrypt(data, secretKey);
// System.out.println("加密后的数据: " + encryptedData);
// 解密数据
SecretKey newSecretKey = stringToKey("tnb12345");
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
System.out.println("解密后的数据: " + decryptedData);
} catch (Exception e) {
e.printStackTrace();
}
}
}
@@ -186,6 +186,26 @@ public class DbUtils{
return query
}
public static BasicDBObject buildSeaweedQuery2(def city = null, def region = null, def name = null, def cat = "district", def status = 0){
BasicDBObject query = new BasicDBObject()
if(city){
query.append("city", city)
}
if(region){
query.append("region", region)
}
if(name){
query.append("name", name)
}
if(cat != null){
query.append("cat", cat)
}
if(status != null){
query.append("status", status)
}
return query
}
/**
* 把小区修改成删除状态
* @param id
@@ -76,7 +76,6 @@ class ElasticUtils {
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@@ -101,6 +100,30 @@ class ElasticUtils {
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 5
def nThread = Runtime.getRuntime().availableProcessors() * 4
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -71,6 +71,10 @@ class MongoUtils {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
}
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
}
def key = db + name + domainUrl + port
if (collectionMap.containsKey(key)) {
return collectionMap[key]
@@ -156,6 +156,22 @@ public class ProvinceData {
return PROVINCE_MAP.get(province);
}
public static String getSimpleProvince(String province){
def prov = null
def newDatas = data + GOVERNMENT_CITY_LIST
for(def key : newDatas.keySet()){
def list = newDatas.get(key)
list << key
if(list.contains(province)){
prov = key
break;
}
}
return prov;
}
/**
* 获取省份下的所有城市
* @param province
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
* 冪帠僅蛌華硊<br>