Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-08-12 15:18:26 +08:00
35 changed files with 2377 additions and 363 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年6月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def fangjiaDists = new HashSet()
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
def today = new Date().getDate()
println "starting 。。。。。。。。。"
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
for(def d : allDists){
fangjiaDists << d._id.toString()
}
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
println "加载建行小区量"
def ccbMaps = null
if(today<=10){
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
println "组装新数据"
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -20,7 +20,7 @@ import java.nio.charset.StandardCharsets
class CcbCommonUtils {
public static def citys_fjw_all = ["上海","长春","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
@@ -442,9 +442,12 @@ class CcbCommonUtils {
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
if(city){
q.put("city", city);
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","荣和怡景园")
// q.put("name","一米阳光一期")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -456,9 +459,9 @@ class CcbCommonUtils {
}
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def rets = new HashSet()
def ids = new HashSet()
def ids = []
for(def d : dbsour){
if(kind){
if(d.address){
@@ -688,6 +691,8 @@ class CcbCommonUtils {
pgsqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
}
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
// if(d['G']){
// def obj = CommonDatautils.findDistrictById(d['G'])
// if(obj){
// d['H'] = obj.name
// }
// }
//})
////def url = "https://sh.anjuke.com"
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
@@ -1289,7 +1299,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240626人工指导价.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){
@@ -1308,9 +1318,9 @@ ExcelUtils.multiThreadingModify(fileName, {d->
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
d['K'] = dist.estate_type2
d['L'] = dist.status
d['M'] = dist.name_cat
// d['H'] = dist.name
}
@@ -1352,24 +1362,57 @@ ExcelUtils.multiThreadingModify(fileName, {d->
//
//})
//def filePath = "D:\\数据\\建行\\房产估值\\指导价\\大庆删除小区.xlsx"
//
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
//ExcelUtils.modify(filePath, { d ->
// if(d['A']){
//
// def obj = CommonDatautils.findDistrictById(d['A'])
// if(obj){
//
// def data = new BasicDBObject()
// data.put("status", 1)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])), new BasicDBObject('$set', data))
//// def obj = CommonDatautils.findDistrictById(d['A'])
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
// if(list){
// if(list.size()==1){
// def dbo = list[0]
// if(dbo){
// def data = new BasicDBObject()
// data.put("region", d['B'])
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
// }
// }else{
// println d['C'] +" 有多个"
// }
// }
//// if(obj){
//// def lng = d['G']
//// def lat =d['H']
////
//// // 高德转百度
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
//// def lng2 = baidu.x
//// def lat2 = baidu.y
////
////
//// def data = new BasicDBObject()
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
//// data.put("lng", lng_new)
//// data.put("lat", lat_new)
////
//// Long dLat2 = lat2 * 10000000000L
//// Long dLng2 = lng2 * 10000000000L
//// data.put("lat2", dLat2)
//// data.put("lng2", dLng2)
////
////// data.put("status", 1)
//// data.put("m_date", new Date())
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
//// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
@@ -0,0 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
if(d['G'] && d['G'] instanceof String){
def obj = CommonDatautils.findDistrictById(d['G'])
if(obj){
d['H'] = obj.name
}
}
})
@@ -32,6 +32,10 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -41,7 +45,9 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -584,6 +590,36 @@ class TradeStatLog {
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def exportTrade2(){
// def startDate = "2024-06-01"
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
if(d['S'] && !d['W']){
def dist = CommonDatautils.findDistrictById(d['S'])
if(!dist.name_cat && dist.name_cat!=0){
d['W'] = "name_cat过滤"
}
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
d['W'] = "小区过滤"
}
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
def map = findDealByCommId(d['B'], d['S'])
def count = map.get(key)
if(count>=2){
d['W'] = "重复"
}
}
println d['A']
})
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
@@ -637,11 +673,39 @@ class TradeStatLog {
return trades
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
ids << r.gujia_bds_mdl_id
}
})
return ids
}
def findDealByCommId(city, commid){
def q = new BasicDBObject()
q.put("match_id", commid)
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
def dbcursor = DEAL_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
return map
}
def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
@@ -146,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
@@ -221,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
@@ -253,7 +253,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
}
@@ -295,7 +295,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
}
@@ -327,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
@@ -356,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
@@ -384,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
@@ -422,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
@@ -452,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
@@ -485,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
@@ -517,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
@@ -552,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
@@ -576,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
@@ -676,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -716,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -987,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -998,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
@@ -1072,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
@@ -1084,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportThirdPartyPrice2(price_date, new_dists) {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1132,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1215,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
@@ -1261,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
@@ -1304,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1319,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1393,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1460,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1501,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1537,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1570,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1609,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1652,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1689,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1737,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1784,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1845,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
@@ -1856,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataForsale(publishDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1913,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataForsale(publishDate,endDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -2098,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
@@ -4653,6 +4653,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4662,7 +4663,7 @@ class CommonDatautils {
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
@@ -4686,7 +4687,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
// q.put("city","芜湖市")
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4712,10 +4713,42 @@ class CommonDatautils {
}
static def queryTownByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "乡镇街道 加载完成"
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("city",1)
@@ -4833,6 +4866,9 @@ class CommonDatautils {
def jsonObject = null
def stop = false
def count = 0
def road = ""
def name = ""
def type = ""
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
@@ -4852,8 +4888,26 @@ class CommonDatautils {
}
format_addr = jsonObject.regeocode.formatted_address
if(addressComponent.streetNumber){
if(addressComponent.streetNumber.street){
road = addressComponent.streetNumber.street
}
if(addressComponent.streetNumber.number){
road = road + addressComponent.streetNumber.number
}
}
if(addressComponent.building){
if(addressComponent.building.name){
name = addressComponent.building.name
}
if(addressComponent.building.type){
type = addressComponent.building.type
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
@@ -4871,6 +4925,9 @@ class CommonDatautils {
map.district = district
map.township = township
map.format_addr = format_addr
map.road = road
map.type = type
map.name = name
return map;
}
@@ -0,0 +1,79 @@
package com.fangjia.script.gj
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class GreenBuild {
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new GreenBuild()
task.run()
}
def run(){
def datas = findAll()
def rets = match(datas)
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
}
def match(datas){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
data.match_id = resultMap.get("id").toString()
data.match_city = resultMap.get("city")
data.match_district = resultMap.get("district")
data.match_name = resultMap.get("name")
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
}else{
rets << [data._id.toString(), data.city, data.district, data.name]
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await();
return rets
}
def findAll(){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
}
}
@@ -12,8 +12,8 @@ class OpenUserBatch {
def task = new OpenUserBatch();
// task.run();
// task.test();
// task.addShenZhenFjtUser();
task.addFujianFjtUser()
task.addShenZhenFjtUser();
// task.addFujianFjtUser()
}
def test(){
@@ -46,20 +46,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252608 + i;
def id = 252893 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
onLineSqlInstance.execute(sql3)
}
}
File diff suppressed because one or more lines are too long
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -17,6 +18,8 @@ import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import java.util.regex.Pattern
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
@@ -28,11 +31,13 @@ class DevProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
static main(args) {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -41,6 +46,7 @@ class DevProcess {
// task.updateCityAndDist()
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址
// task.addDevAddres3()
// task.matchDevAndAddAddress()
// task.stat()
@@ -56,6 +62,15 @@ class DevProcess {
// task.matchDevTownship()//添加乡镇街道
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
// task.matchNewCrawlDev()
// println DateUtils.add("2024-7-24", 7);
// task.addKfc()
// task.devToPark() //开发区下辖园区
task.checkTown()
}
@@ -334,15 +349,22 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
if(!d['Q'] && d['L']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d->
if(!d['Q'] && d['A']){
def q = new BasicDBObject()
q.put("name", d['L'])
q.put("city",d['G'])
// q.put("name", d['A'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['A']]))
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['Q'] = obj._id.toString()
d['R'] = obj.name
d['C'] = obj._id.toString()
d['D'] = obj.name
if(obj.name == d['A']){
d['E'] = "相同"
}else{
d['E'] = "包含"
}
}
}
},0);
@@ -509,6 +531,38 @@ class DevProcess {
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def addDevAddres3(){
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
if(d['A']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
d['C'] = obj._id.toString()
def q2 = new BasicDBObject()
q2.put("name", d['A'])
q2.put("address", d['B'])
q2.put("type","地址")
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(!sameObj){
index++
def udata = new BasicDBObject()
udata.put("province", obj.province)
udata.put("city", obj.city)
udata.put("district", obj.district)
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address", d['B'])
DEV_MAPPING_CONN.save(udata)
}
}
}
},0)
println index
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def stat(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
def total = 0
@@ -777,4 +831,128 @@ class DevProcess {
})
}
def matchNewCrawlDev(){
def query = new BasicDBObject()
query.put("match_id", new BasicDBObject('$exists', false))
def list = kfq_COLL.find(query).toList()
for(def d : list){
def q = new BasicDBObject()
// q.put("name", d.name)
q.put('alias', new BasicDBObject('$in',[d.name]))
def obj = development_zone_CONN.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("match_id", obj._id.toString())
udata.put("match_name", obj.name)
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addKfc(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
if(d['I']){
if(d['I'] == "添加"){
def udata = new BasicDBObject()
udata.put("create_time" , new Date())
udata.put("industry",d['D'])
if(d['E'].contains("国家级")){
udata.put("level" ,"国家级")
}else if(d['E'].contains("省级")){
udata.put("level" ,"省级")
}
udata.put("name" ,d['F'])
udata.put("province" ,d['G'])
udata.put("type" ,d['H'])
udata.put("city" ,d['B'])
udata.put("district" ,d['J'])
development_zone_CONN.save(udata)
}else{
def q = new BasicDBObject()
q.put("name", d['I'])
def obj = development_zone_CONN.findOne(q)
if(obj){
def alias = []
if(obj.alias){
alias = obj.alias
}
if(!alias.contains(d['F'])){
alias << d['F']
}
if(alias && alias.size()!=obj.alias.size()){
def updateData = new BasicDBObject()
updateData.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
}
}
}
}
})
}
def devToPark(){
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
def rets = []
rets << ["开发区名称", "下辖街道", "城市", "区域"]
for(def d : datas){
if(d['E']){
def arrs = d['E'].split(",")
def q = new BasicDBObject().append("name", d['B'])
def obj = development_zone_CONN.findOne(q)
if(obj){
for(def park : arrs){
rets << [d['B'], park, obj.city, obj.district]
}
}
}
}
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
}
def checkTown(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d->
if(!d['E']){
if(d['C'] && d['D']){
// if(d['A']=="长春高新技术产业开发区"){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
if(jsonObject1) {
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
def township = jsonObject1.get("township")
def district = jsonObject1.get("district")
d['E'] = "接口"
if(district && district!='null'){
d['F'] = district
}
if(township && township!='null'){
d['G'] = township
}
d['H'] = jsonObject1.get("townCode")
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
d['I'] = jsonObject1.get("provinceCode")
}
}
// }
}
// def q = new BasicDBObject()
// q.put("city", d['C'])
//// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
// q.put("type", "乡镇街道")
// q.put("street", d['B'])
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
// if(obj){
// d['E'] = "可入库2"
// d['F'] = obj.district
// }
}
}
},1)
}
}
@@ -0,0 +1,277 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
o.put("city", d['C'])
o.put("district", d['D'])
o.put("name", d['A'])
o.put("level", d['E'])
o.put("type", d['F'])
if(d['G']){
o.put("township", d['G'])
}
if(d['H']){
def arr = d['H'].split(",")
if(arr){
o.put("alias", arr)
}
}
def id = addMongoDB(o)
o.remove("_id")
o.id = id.toString()
o.remove("create_time")
addEs(o, "192.168.0.89", "development_zone")
addEs(o, "192.168.0.52", "development_zone")
d['I'] = "已完成"
}else{
println d['A'] +" 已存在"
}
}
})
}
def addMongoDB(o){
o.put("create_time", new Date())
development_zone_100_CONN.save(o)
DEVELOPMENT_ZONE_43_COLL.save(o)
return o._id
}
def addEs(map, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
q.put("township", d['C'])
}else if(d['B'] == "地址"){
q.put("address", d['C'])
}else if(d['B'] == "园区"){
q.put("park", d['C'])
}
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
if(!mappingObj){
def o = new BasicDBObject()
o.put("province", obj.province)
o.put("city", obj.city)
o.put("district", obj.district)
o.put("name", obj.name)
o.put("type", d['B'])
if(d['B'] == "乡镇街道"){
o.put("township", d['C'])
}else if(d['B'] == "地址"){
o.put("address", d['C'])
}else if(d['B'] == "园区"){
o.put("park", d['C'])
}
def date = new Date()
o.put("c_date", date)
o.put("version", date.getTime())
DEV_MAPPING_100_CONN.save(o)
o.id = o._id.toString()
o.remove("_id")
o.remove("c_date")
addEs(o, "192.168.0.89", "dev_mapping")
try{
addEs(o, "192.168.0.52", "dev_mapping")
}catch(Exception e){
println "52 es error: "+e.printStackTrace()
}
d['D'] = "已完成"
}else{
d['D'] = "存在"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
println d['A'] +" 不存在"
}
}
},2)
}
def updateHasGov(province, city, district, street, hasGov){
def q = new BasicDBObject()
q.put("province", province)
q.put("city", city)
q.put("district", district)
q.put("street", street)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
//修改43
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
//修改89 administrative_mapping
update89EsAdministrativeMapping(province, city, district, street, hasGov)
//修改89 administrative_mapping
update52EsAdministrativeMapping(province, city, district, street, hasGov)
}else{
println province+" "+city+" "+district+" "+street+" 没找到"
}
//修改100 community
//修改89 community
//修改52 community
}
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
}
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
task.run()
// task.run()
task.fenQiProcess()
}
def run(){
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
}
})
}
def fenQiProcess(){
def q = new BasicDBObject()
q.put("cat","district")
q.put("status", 0)
q.put("master_district_name", 'master')
def dbcursor = DbUtils.seaweed.find(q)
def rets = []
for(def d : dbcursor){
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
}
for(def r : rets){
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def lastObj = null
for(def j : rets){
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
lastObj = j;
break
}
}
if(lastObj){
println r
def udata = new BasicDBObject()
udata.put("master_district_name","")
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
// task.addOldMatch()
task.addOldMatch()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
task.runMatch()
// task.runMatch()
}
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
@@ -35,21 +35,52 @@ class PoiLocation {
static main(args) {
def task = new PoiLocation();
task.run();
// task.run();
task.regionAndStreetProcess()
}
def run(){
def citys = ["芜湖市","重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"]
def citys = [""]
citys = citys.reverse()
for(def city : citys){
findPois(city)
}
}
def regionAndStreetProcess(){
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
for(def city : citys){
def datas = findAllPoisByCity(city)
process(datas)
}
}
def findAllPoisByCity(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("run_status", new BasicDBObject('$ne', 1))
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
keys.put("_id", 1)
@@ -87,6 +118,32 @@ class PoiLocation {
return rets;
}
def process(datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
dataRun(d)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
@@ -190,6 +247,33 @@ class PoiLocation {
return map;
}
def dataRun(d){
def regions = getFullRegion(d)
if(d.location) {
//判断区域否在围栏里
def regionFlag = null
if (d.district) {
for (def rs : regions) {
if (rs.shape && rs.shape.coordinates) {
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if (regionFlag) {//在围栏里
//判断township是否在统计局中
// if(d.township){
// town = getSimilarTown(regions, d.city, d.township)
// }
}else{//不在转栏里,重新搜索
println(d)
}
}
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
@@ -0,0 +1,101 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThardParthAnalyBatch(), args)
}
// static main(args) {
// def task = new ThardParthAnalyBatch()
// task.run()
// }
@Override
void process(Object args) {
run()
}
def run(){
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
println "统计月份:"+sDate+" 查询更新时间:"+mdate
def crawlTotalCount = findAllEsDataCount(mdate)
println "抓取第三方小区量:"+crawlTotalCount
def synchTotalCount = findAllMongoDatasCount(sDate)
println "同步到第三方库小区量: "+synchTotalCount
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
def matchTotalCount = findMatchSuccessCount(sDate)
println "第三方小区匹配成功量:"+matchTotalCount
def noMatchTotalCount = findNoMatchCount(sDate)
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount);
}
def findAllEsDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllMongoDatasCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
return thirdPartyColl.count(q)
}
def findAllMongoDatasHavePricesCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
return thirdPartyColl.count(q)
}
def findMatchSuccessCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
return thirdPartyColl.count(q)
}
def findNoMatchCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
return thirdPartyColl.count(q)
}
def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
def udata = new BasicDBObject()
udata.put("crawl_total", crawlTotalCount)
udata.put("synch_total", synchTotalCount)
udata.put("synch_have_price_total", synchHavePriceTotalCount)
udata.put("match_total", matchTotalCount)
udata.put("no_match_total", noMatchTotalCount)
udata.put("c_date", new Date())
if(obj){
udata.put("m_date", new Date())
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
}else{
udata.put("s_date", sDate)
THIRD_PARTY_STAT.save(udata)
}
}
}
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -26,6 +30,9 @@ class TownMiddleProcess {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
@@ -67,7 +74,15 @@ class TownMiddleProcess {
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
// addAliasFromExcel()
//增加简称及简拼
// addSimpleName()
//根据民政部数据修改行政区划中间表
// mzbDistrictUpate()
// addCity()
}
@@ -422,14 +437,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -929,4 +946,222 @@ class TownMiddleProcess {
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
def addAliasFromExcel(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
def q = new BasicDBObject("type", "乡镇街道")
q.put("city", row['A'])
q.put("street", row['D'])
if(row['B']){
q.put("district", row['B'])
}
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
row['F'] = '是'
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(row['C'])){
alias << row['C']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
def updateQuery = new BasicDBObject("_id", obj._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
})
}
def addSimpleName(){
def q = new BasicDBObject()
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
def prov = ProvinceData.getSimpleProvince(d.province)
if(prov){
def udata = new BasicDBObject()
udata.put("province_short", prov)
def updateQuery = new BasicDBObject("_id", d._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
// def city = d.city.replace("市","").replace("区","").replace("县","")
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// if(d.district && d.district.size()>=3){
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
// udata.put("district_short", region)
// }
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// def city = RegionData.formatCity(d.city)
// if(city && d.mzb_district_name){
// def region = RegionData.formatRegion(city, d.mzb_district_name)
// if(region){
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// udata.put("district_short", region)
//
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// }
// }
}
}
def mzbDistrictUpate(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
if(!row['I']){
def updateType = row['E']
def prov = row['B']
def newStreet = row['G']
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
}
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
if(row['F']){
def dbo = null
def code = row['F'].toString()
if(!code.contains(" ")){
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
}
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}else if(updateType == "撤街道设街道"){
row['K'] = "人工处理"
}else if(updateType == "隶属关系变更"){
row['K'] = "人工处理"
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
row['K'] = "人工处理"
}else if(updateType =='撤销乡'){
if(!row['F']){
row['K'] = "人工处理"
}
}
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
def dbo = findStreets(prov, newStreet, null)
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在-编码不同"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
// addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}
})
}
def addCity(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
def citycode = null
if(row['F']){
citycode = row['F'].toString().substring(0, 4)
}else if(row['C']){
citycode = row['C'].toString().substring(0, 4)
}
if(citycode){
def q = new BasicDBObject()
q.put("mzb_city_code", citycode+"00")
q.put("type", "城市")
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(dbo){
row['O'] = dbo.city
}
}
})
}
def findStreets(prov, newStreet, streetCode){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
q.put("province", new BasicDBObject('$regex', provPattern))
q.put("street", newStreet)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def list = []
def dbo = null
for(def d : dbcursor){
list << d
}
if(list && list.size()==1){
dbo = list[0]
}else if(list && list.size()>1){
if(streetCode){
for(def d : list){
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
dbo = d
break;
}else if(streetCode.contains(d.pid)){
dbo = d
break;
}
}
}
}
return dbo
}
def addAlias(id, alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("operator","高娟-批量")
def updateQuery = new BasicDBObject("_id", id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -9,12 +10,14 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TownshipStrandUpdate {
@@ -26,13 +29,26 @@ class TownshipStrandUpdate {
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
task.updateTown()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// task.updateSpecCityTown()
}
@@ -43,21 +59,21 @@ class TownshipStrandUpdate {
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["鉗�","벳덱","柯소鏤","괏땍","해뽈돎","읜렌","꾕鹵","蠟소왯","뷜彊","饉憩","넓돠"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
citys = citys.reverse()
for(def city : citys){
//
if(["中山","东莞"].contains(city)){
continue
}
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
// if(["中山","东莞"].contains(city)){
// continue
// }
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
// if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
}
// }
}
}
@@ -162,6 +178,10 @@ class TownshipStrandUpdate {
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
regions = []
regions << fullCity.city
}
def gaodeMap = null
def town = null
def mapTown = null
@@ -217,6 +237,10 @@ class TownshipStrandUpdate {
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}else{
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
mapRegion = mapCity
}
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
@@ -331,7 +355,7 @@ class TownshipStrandUpdate {
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region)){
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
names << r.district
}
}
@@ -381,7 +405,7 @@ class TownshipStrandUpdate {
}
}
}
if(district || count>10){
if(township || district || count>10){
stop = true
}else{
count ++
@@ -397,4 +421,212 @@ class TownshipStrandUpdate {
map.format_addr = format_addr
return map;
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
d['AB'] = "已处理"
}
}
}
})
}
def cleanTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['AC']){
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dbo && dbo.township){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
}
}
})
}
def checkTownFromMongo(){
def citys = CommonDatautils.findAllCity()
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
citys = citys.reverse()
def index = 0
for(def city : citys) {
index++
if (["中山", "东莞"].contains(city)) {
continue
}
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
for (def d : dists) {
if (d.township) {
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def town = getSimilarTown(regions, fullCity, d.township)
if (!town) {
town = findStreet(d.city, d.city, d.township)
if (!town) {
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
println d.city + " " + d.region + " " + d.township + " " + d.name
}
}
}
}
println index + " " + city + " 完成"
}
}
def findStreet(city, district, street){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
q.put("city", new BasicDBObject('$regex', provPattern))
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
if(list){
if(list.size()==1){
return list[0]
}else{
println city+ " "+district+" "+street+" 有多个"
}
}
return null
}
def updateTownMapping(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['F']){
// def udata = new BasicDBObject()
// udata.put("mzb_district_name", d['F'])
//
// def updateQuery = new BasicDBObject()
// updateQuery.put("city", d['C'])
// updateQuery.put("district", d['D'])
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
def city = RegionData.formatCity(d['C'])
if(city){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def updateQuery = new BasicDBObject()
updateQuery.put("category", "region")
updateQuery.put("city", city)
updateQuery.put("name", region)
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
if(list && list.size()==1){
def dbo = list[0]
def query = new BasicDBObject()
query.put("_id", dbo._id)
// def pattern = dbo.pattern
// if(pattern.contains(d['F'])){
// pattern.remove(d['F'])
// }
def udata = new BasicDBObject()
udata.put("org_name", d['F'])
// udata.put("pattern",pattern)
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
}else{
println city +" "+region+" "+list.size()
}
}
}
}
})
}
def updateDevMapping(){
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("district", d['B'])
q.put("name",d['C'])
q.put("type", "乡镇街道")
q.put("township", d['D'])
def list = dev_MAPPING_COLL_100.find(q).toList()
if(list && list.size()==2){
def obj = list[0]
def removeQuery = new BasicDBObject()
removeQuery.put("_id", obj._id)
dev_MAPPING_COLL_100.remove(removeQuery)
}else{
println q
}
})
}
def updateSpecCityTown(){
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
// def citys = ["中山"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
ExcelUtils.modify(fileName,{d->
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
if(list.size()>1){
println d['A'] +" "+d['C'] + " 有多个"
}else if(list.size()==1){
def dbo = list[0]
def town = null
// if(d['X']){
// town = d['X']
// }else
if(d['R']){
town = d['R']
}
if(!town && d['M']){
town = d['M']
}
def region = RegionData.formatRegion(d['A'], town)
if(region){
def q = new BasicDBObject()
q.put("region", region)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", town)
udata.put("region", region)
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
}else{
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
}
}
})
}
}
}
@@ -53,31 +53,33 @@ class FjwBigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
def fangjiaDists = new HashSet()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
for(def d : dists){
fangjiaDists << d._id.toString()
}
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
}
if(today > 10){
// if(today > 10){
delPgData()
saveToPgSql(rows)
}
// }
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -515,13 +515,12 @@ class FjwBigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
@@ -72,7 +72,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
@@ -93,10 +93,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
def houseIdsfilter = null;//findAllHouseIdsByCity()
for(def city : citys){
index++
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
@@ -492,8 +493,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -918,11 +919,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(){
def findAllHouseIdsByCity(cityid){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
filter.put(it.gujia_sell_id)
});
@@ -48,19 +48,24 @@ class FjwGuiDingPriceBatch extends BaseBatch{
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
rows.addAll(rets)
println city + " " + rets.size()
}
}
println "【房价网内部】人工指导价总计:"+rows.size()
saveToPgSql(rows)
}
@@ -118,11 +123,15 @@ class FjwGuiDingPriceBatch extends BaseBatch{
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(priceIdsfilter.mightContain(ps._id.toString())){
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
continue
}
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
@@ -1,7 +1,11 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import groovy.sql.Sql
import java.nio.charset.StandardCharsets
/**
* 房价网内部使用
*/
@@ -35,5 +39,14 @@ class FjwPgsqlCommUtils {
sqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
filter.put(it.gujia_fsid)
});
return filter
}
}
@@ -35,9 +35,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
@@ -50,20 +47,23 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def stop = false
def month = -1
while(!stop){
if(sDate<202004){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
}
month = month -1
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
if(sDate == 201912){
stop = true;
}
}
runDatas(citys)
//ͬ²½ÀúÊ·Êý¾Ý
// def stop = false
// def month = -1
// while(!stop){
// if(sDate>202404 && sDate<202407){
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
// println weekend.toString() +" "+sDate
// runDatas(citys)
// }
// month = month -1
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
// if(sDate == 201912){
// stop = true;
// }
// }
}
def runDatas(citys){
@@ -389,139 +389,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
@@ -33,7 +33,7 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\分期小区对比结果.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
@@ -167,14 +167,14 @@ class AddPriceTest {
}
def addCcbPrice(filepath){
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['A']){
// def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(d['AJ']){
def cityId = cityCodeDataMap.get(d['AK'])
def map = findAvgPrice(d['AJ'], cityId)
if(map){
// d['AK'] = map.avg_price
d['P'] = map.calc_type
d['F'] = map.avg_price
// d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
@@ -183,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -272,23 +272,23 @@ class AddPriceTest {
ExcelUtils.multiThreadingModify(path,{d->
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
def dist = CommonDatautils.findDistrictById(d['AJ'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['D'] = dist.status
d['E'] = dist.name_cat
d['F'] = dist.estate_type2
d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
d['H'] = dist.master_district_name
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
// d['H'] = dist.master_district_name
// d['N'] = dist.region
// def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
// if(dbo){
// d['D'] = dbo.avg_price
//// d['X'] = dbo.cal_type
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['G'] = dbo.avg_price
// d['X'] = dbo.cal_type
// d['E'] = dbo.avg_price_2b
// }
}
}
// }
// }
@@ -27,8 +27,8 @@ class QiliangTest {
// task.run()
def data = [
'address': '宁德市福安市中兴西路16号',
'type': '2' //1:对公 0:对私
'address': '江阴市澄杨路77号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.xiamen
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils89 {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
// connectionProperties.setProperty("user","developer");
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
connectionProperties.setProperty("user","elastic");
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
}
})
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
File diff suppressed because one or more lines are too long
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
//查找出所有未进行匹配的数据
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -0,0 +1,69 @@
package com.fangjia.script.util
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
public class DESUtil {
private static final String ALGORITHM = "DES";
// 生成密钥
public static SecretKey generateKey() throws Exception {
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
keyGenerator.init(56); // DES密钥长度为56位
return keyGenerator.generateKey();
}
// 将密钥转换为字符串
public static String keyToString(SecretKey secretKey) {
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
}
// 将字符串转换为密钥
public static SecretKey stringToKey(String keyStr) {
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
}
// DES加密
public static String encrypt(String data, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
byte[] encryptedData = cipher.doFinal(data.getBytes());
return Base64.getEncoder().encodeToString(encryptedData);
}
// DES解密
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.DECRYPT_MODE, secretKey);
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
byte[] decryptedData = cipher.doFinal(decodedData);
return new String(decryptedData);
}
public static void main(String[] args) {
try {
// 生成密钥
// SecretKey secretKey = generateKey();
// String keyStr = keyToString(secretKey);
// System.out.println("密钥: " + keyStr);
//
//// // 加密数据
// String data = "高娟";
// String encryptedData = encrypt(data, secretKey);
// System.out.println("加密后的数据: " + encryptedData);
// 解密数据
SecretKey newSecretKey = stringToKey("tnb12345");
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
System.out.println("解密后的数据: " + decryptedData);
} catch (Exception e) {
e.printStackTrace();
}
}
}
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
* 冪帠僅蛌華硊<br>