From 9b7351d9db484b3128450d6a10110715dd5727dc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 May 2024 17:32:25 +0800 Subject: [PATCH 01/29] =?UTF-8?q?=E5=B0=86=E4=B8=8A=E4=B8=8A=E4=B8=AA?= =?UTF-8?q?=E6=9C=88=E6=99=9A=E6=8A=93=E7=9A=84=E6=88=90=E4=BA=A4=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E4=BF=AE=E6=94=B9=E4=B8=BA=E4=B8=8A=E6=9C=881?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 122 ++++++++++-------- .../ccbevaluation/CcbCommonUtils.groovy | 12 ++ .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- 3 files changed, 83 insertions(+), 53 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 210e810a2..d7bf3a8a3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql +import org.bson.types.ObjectId import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -62,60 +63,60 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - - println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ - addDealPriceMatch(city); - } +// +// println "curDate:"+curDate +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } // updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 -// for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) -// println "历史id加载量:"+dealIds.size() -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -//// delPgData() -// saveToPgSql(rows) -// -//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") -//// saveToPgSql2(datas) + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) + println "历史id加载量:"+dealIds.size() + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + +// delPgData() + saveToPgSql(rows) + +// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +// saveToPgSql2(datas) } def readTxt(def fileName){ @@ -682,7 +683,13 @@ class BigdataDealBatch extends BaseBatch{ } def updateCrawlTradeDate(){ - + def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20" + def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd") + def objectId = dateToObjectId(date) + def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2) + def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1) + println(objectId) + CcbCommonUtils.updateTrades(startDate, endDate, objectId) } @@ -822,4 +829,15 @@ class BigdataDealBatch extends BaseBatch{ println "end: "+new Date() } + + + def dateToObjectId(Date date) { + // 获取时间戳(单位:秒) + def timestamp = (date.time / 1000) as int + def hexTimestamp = Integer.toHexString(timestamp) + + // 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值 + def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000' + return new ObjectId(objectIdStr) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 84f840c02..061937568 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -711,4 +711,16 @@ class CcbCommonUtils { return map } + public static def updateTrades(startDate, endDate, cDate){ + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("_id", new BasicDBObject('$gte', cDate)) + def count = DEAL_PRICE_COLL.count(q) + if(count>0){ + def udata = new BasicDBObject() + udata.put("trade_date", endDate) + DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata)) + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 9ad3f9a31..691a3f859 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-20") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) From f52eed9cce749013cb92b98195dba6cee6356c94 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Wed, 22 May 2024 17:54:07 +0800 Subject: [PATCH 02/29] =?UTF-8?q?=E5=8C=97=E4=BA=A4=E6=89=80=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/ComprehensiveQueryController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java index 4d264b835..145926d26 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java @@ -233,10 +233,12 @@ public class ComprehensiveQueryController { communityAvgPriceParams.put("city", model.getCity()); communityAvgPriceParams.put("address", model.getAddress()); communityAvgPriceParams.put("category", model.getCategory()); - communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); +// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); + communityAvgPriceParams.put("suggestName", String.valueOf(model.getName())); + communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"); JSONObject matchResponse = null; try { - matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8"); + matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8"); // matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8"); } catch (HttpRequestException e) { From 80110fbee847334bd3a026bb06f12b3e1d9c0dd8 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Wed, 22 May 2024 18:42:29 +0800 Subject: [PATCH 03/29] =?UTF-8?q?=E5=8C=97=E4=BA=A4=E6=89=80=E4=BB=B7?= =?UTF-8?q?=E6=A0=BC=E6=94=B9=E6=88=902B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ComprehensiveQueryController.java | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java index 145926d26..55872ef9d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java @@ -258,9 +258,21 @@ public class ComprehensiveQueryController { DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district)); if (null != statBo) { if (statBo.getDistrictStat() != null) { - dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); + // 有2B价格,使用2B价格 + Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b(); + if (avgPrice2b != null && avgPrice2b > 0) { + dto.setAvgPrice(avgPrice2b); + } else { + dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); + } } else if (statBo.getVillaDistrictStat() != null) { - dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); + // 有2B价格,使用2B价格 + Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b(); + if (avgPrice2b != null && avgPrice2b > 0) { + dto.setAvgPrice(avgPrice2b); + } else { + dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); + } } } @@ -640,10 +652,18 @@ public class ComprehensiveQueryController { if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { int factorPrice = evaluation.getAvgPrice(); int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); - Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95); - Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05); + // 有2B价格用2B价格,没有用2C + Integer minPrice = 0; + Integer maxPrice = 0; + if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) { + minPrice = (int) (evaluation.getAvgPrice2b() * 0.95); + maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05); + } else { + minPrice = (int) (evaluation.getAvgPrice() * 0.95); + maxPrice = (int) (evaluation.getAvgPrice() * 1.05); + } // 偏离度超过5%不进行调整 - if (minPrice <= avgPrice && avgPrice <= maxPrice) { + if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) { evaluation.setAvgPrice(avgPrice); evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); } @@ -659,7 +679,11 @@ public class ComprehensiveQueryController { districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); - districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); + if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) { + districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b()); + } else { + districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); + } // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -884,7 +908,11 @@ public class ComprehensiveQueryController { otherPriceRets = new HashMap(); } - otherPriceRets.put("fjw_price", evaluation.getAvgPrice()); + if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){ + otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b()); + }else { + otherPriceRets.put("fjw_price", evaluation.getAvgPrice()); + } Map levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit()); Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit); From caf17ed9e46d07d7c6c38ea9a779a18dfa3723c6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 May 2024 10:26:04 +0800 Subject: [PATCH 04/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=88=BF=E6=BA=90?= =?UTF-8?q?=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 7 ++- .../fangjia/script/ccbevaluation/Test.groovy | 60 +++++++++++++------ .../script/gj/data_handle/DevProcess.groovy | 44 ++++++++++---- .../FjwThirdPartyPriceBatch.groovy | 2 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 30 +++++----- 5 files changed, 94 insertions(+), 49 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 061937568..343189b24 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -15,9 +15,10 @@ import groovy.sql.Sql class CcbCommonUtils { - // - public static def citys_fjw_all = [ "乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] - public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] + //"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都", + public static def citys_fjw_all = [ "沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + //"乌鲁木齐", + public static def citys_all = ["上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index c42ab1349..7978ddbeb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,31 +1289,53 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx" - -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['J']){ -// def region = RegionData.formatRegion(d['B'], d['C']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格不作为指导价录入5-20.xlsx" +// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['J']){ +//// def region = RegionData.formatRegion(d['B'], d['C']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") +//// if(dist){ +//// d['G'] = dist.region +//// d['H'] = dist._id.toString() +//// } +//// } +// if(!d['A']){ +// def dist = CommonDatautils.findDistrictById(d['J']) // if(dist){ -// d['G'] = dist.region -// d['H'] = dist._id.toString() +// d['A'] = dist.city +// d['B'] = dist.region +// d['C'] = dist.name +// d['K'] = dist.estate_type2 +// d['L'] = dist.status +// d['M'] = dist.name_cat // } // } - if(!d['A']){ - def dist = CommonDatautils.findDistrictById(d['J']) - if(dist){ - d['A'] = dist.city - d['B'] = dist.region - d['C'] = dist.name - d['K'] = dist.estate_type2 - d['L'] = dist.status - d['M'] = dist.name_cat +// } +//},0) + + +def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" +ExcelUtils.multiThreadingModify(filePath, { d -> + if(d['B']){ + + def obj = CommonDatautils.findDistrictById(d['B']) + if(obj){ + def markMap = obj.mark_map + if(!obj.mark_map){ + markMap = [:] } + def data = new BasicDBObject() + markMap.put("exclusive", "是") + data.put("mark_map", markMap) + data.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) } } -},0) + +}) + //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" //ExcelUtils.multiThreadingModify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index aa7425f23..1098539f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -30,7 +30,7 @@ class DevProcess { // task.run() // task.parseCityAndDist() // task.devSplitRoad() - task.devNameSplitRoad() +// task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -44,6 +44,7 @@ class DevProcess { // task.addNewDev() // task.createDevMapping() // task.deleteAddr() + task.deleteAddr2() // task.synchDevTo43() } @@ -163,11 +164,11 @@ class DevProcess { } def devSplitRoad(){ - def i = 10; - while(i<11){ + def i = 0; + while(i<1){ def fileName = "/home/goojia/gj/company-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\company-${i}.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\56万企业地址\\省没出值地方.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ @@ -193,8 +194,8 @@ class DevProcess { } } }*/ - if(d['C'] && d['B']!='河北省'){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C']) + if(d['C'] && d['B']!='河北省' && !d['J']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) if(jsonObject){ def street = jsonObject.get("street") def streetNumber = jsonObject.get("streetNumber") @@ -234,7 +235,7 @@ class DevProcess { def devNameSplitRoad(){ def fileName = "/home/goojia/gj/company.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> if(d['P'] && d['P']=="路正常" && !d['O']){ @@ -548,6 +549,26 @@ class DevProcess { }); } + def deleteAddr2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北开发区删除地址.xlsx",{d-> + if(d['E']){ + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['E'])) + def obj = development_zone_CONN.findOne(q) + if(obj){ + def q2 = new BasicDBObject() + q2.put("name", obj.name) + q2.put("address", d['A']+d['B']) + def sameObj = DEV_MAPPING_CONN.findOne(q2) + if(sameObj){ + DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id)) + } + } + } + + }); + } + def synchDevTo43(){ def dbcursor = development_zone_CONN.find() def rets = [] @@ -559,16 +580,16 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx",{d-> - if(d['P'] && d['P']=="路正常" && d['O']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万企业地址\\56万需修改地址.xlsx",{d-> + if(d['L'] && d['L']=="路正常" && d['K']){ def q = new BasicDBObject() - q.put("name", d['O']) + q.put("name", d['K']) def obj = development_zone_CONN.findOne(q) if(obj){ d['Q'] = obj._id.toString() d['R'] = obj.name - def address = d['M']+d['N'] + def address = d['I']+d['J'] q.put("address", address) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ @@ -585,6 +606,7 @@ class DevProcess { udata.put("type", "地址") udata.put("address", address) DEV_MAPPING_CONN.save(udata) + println obj.name +" "+address } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index 76a828232..13b557f07 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -49,7 +49,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = ["乌鲁木齐"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) def stop = false def month = -1 while(!stop){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 29214f73f..93a106c85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,10 +33,10 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\对标补充数据.xlsx" -// addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\苏州价格.xlsx" + addGXDPrice(filePath) // addCcbPrice(filePath) - addThirdPrice(filePath) +// addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() @@ -190,11 +190,11 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['G']) { - def dist = CommonDatautils.findDistrictById(d['G']) + if (d['C']) { + def dist = CommonDatautils.findDistrictById(d['C']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) - d['Z'] = thirdPrice + d['O'] = thirdPrice } } }) @@ -227,12 +227,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['K']) { - def dist = CommonDatautils.findDistrictById(d['C']) + if (!d['L']) { + def dist = CommonDatautils.findDistrictById(d['E']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['K'] = gxdPrice + d['L'] = gxdPrice } } }) @@ -256,16 +256,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['W']){ - if(d['G'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['G']) + if(!d['N']){ + if(d['C'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['C']) if(dist){ // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ - d['W'] = dbo.avg_price - d['X'] = dbo.cal_type -// d['J'] = dbo.avg_price_2b +// d['W'] = dbo.avg_price +// d['X'] = dbo.cal_type + d['N'] = dbo.avg_price_2b } } } From 387e847c02f49a5f00115ca0785d12f9ffcea67d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 23 May 2024 15:20:28 +0800 Subject: [PATCH 05/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/dal/seaweed/model/District.java | 10 +-- .../open/controllers/PropertyController.java | 69 +++++++++++++++---- .../service/datastore/DistrictService.java | 4 +- .../datastore/DistrictServiceImpl.java | 4 +- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 2bb00ca36..00d35fa46 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -187,7 +187,7 @@ public class District implements SeaweedEntity { private double score; // 小区性质 private String landProperty; - private String masterDistrictNname; + private String masterDistrictName; private String ip; private String addrType; private boolean selected; @@ -309,12 +309,12 @@ public class District implements SeaweedEntity { } @Column(name = "master_district_name") - public String getMasterDistrictNname() { - return masterDistrictNname; + public String getMasterDistrictName() { + return masterDistrictName; } - public void setMasterDistrictNname(String masterDistrictNname) { - this.masterDistrictNname = masterDistrictNname; + public void setMasterDistrictName(String masterDistrictName) { + this.masterDistrictName = masterDistrictName; } public void setCategory(String category) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 81a281610..875390efc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -2524,14 +2524,32 @@ public class PropertyController { if (district.getAlias() != null) { districtNames.addAll(district.getAlias()); } - } - List dealPrices = null; - for (String name : districtNames) { - dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model); - if (!dealPrices.isEmpty()) { - break; + final String masterDistrictName = district.getMasterDistrictName(); + if (StringUtils.isNotBlank(masterDistrictName)) { + List districts; + if ("master".equals(masterDistrictName)) { + // 所有的分期数据 + districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName()); + } else { + // 根据主小区名查询所有名称 + districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName); + } + if (CollectionUtils.isNotEmpty(districts)) { + for (District districtTemp : districts) { + districtNames.add(districtTemp.getName()); + } + } } } +// List dealPrices = new ArrayList(); + // 将主名与映射别名查得成交加入 存在不能查出最新成交问题 +// for (String name : districtNames) { +// List dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model); +// if (CollectionUtils.isNotEmpty(dealPricesTempList)) { +// dealPrices.addAll(dealPricesTempList); +// } +// } + List dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), districtNames, model.getMax() * 5, model); if (model.getSize() != null) { Collections.sort(dealPrices, new Comparator() { @Override @@ -2580,15 +2598,9 @@ public class PropertyController { } deal = new LinkedHashMap(); dealPrice.setId(null); - if (district != null) { - deal.put("city", district.getCity()); - deal.put("district", district.getRegion()); - deal.put("name", district.getName()); - } else { - deal.put("city", dealPrice.getCity()); - deal.put("district", dealPrice.getRegion()); - deal.put("name", dealPrice.getDistrictName()); - } + deal.put("city", dealPrice.getCity()); + deal.put("district", dealPrice.getRegion()); + deal.put("name", dealPrice.getDistrictName()); if (dealPrice.getArea() == null) { deal.put("size", ""); } else { @@ -3501,6 +3513,33 @@ public class PropertyController { return null; } + private List findDealPricesByNames(String city, String region, List names, int max, + DealPriceModel model) { + List params = new ArrayList(); + params.add(new QueryParam("city", city)); + params.add(new QueryParam("region", region)); + params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names)); + if (model.getRoom() != null) { + params.add(new QueryParam("room", model.getRoom())); + } + if (model.getHall() != null) { + params.add(new QueryParam("hall", model.getHall())); + } + // 18年之后抓住的数据没有score字段 +// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l)); + List dealPrices = dealPriceService.findDealPrices(params, max); + List distinctDealPrices = new ArrayList(); + for (DealPrice dealPrice : dealPrices) { + if (dealPrice.getAvgPrice() == null) { + dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea())); + } + if (!checkExists(distinctDealPrices, dealPrice)) { + distinctDealPrices.add(dealPrice); + } + } + return distinctDealPrices; + } + private List findDealPricesByName(String city, String region, String name, int max, DealPriceModel model) { List params = new ArrayList(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java index 8cd9fc3f9..68588d8e5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java @@ -540,10 +540,10 @@ public interface DistrictService extends ModeService { * * @param city * @param region - * @param masterDistrictNname + * @param masterDistrictName * @return */ - List findSlaves(String city, String region, String masterDistrictNname); + List findSlaves(String city, String region, String masterDistrictName); /** * 分期操作时设置此小区为人工操作标识 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index 31b588538..189508ed8 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService { } @Override - public List findSlaves(String city, String region, String masterDistrictNname) { + public List findSlaves(String city, String region, String masterDistrictName) { List params = new ArrayList(); params.add(new QueryParam("visible", 0)); params.add(new QueryParam("city", city)); params.add(new QueryParam("region", region)); - params.add(new QueryParam("masterDistrictNname", masterDistrictNname)); + params.add(new QueryParam("masterDistrictName", masterDistrictName)); return districtDao.findAllByParams(params); } From db9ccfc3c0db7ead53c557719b98d91e45f52524 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 May 2024 18:22:23 +0800 Subject: [PATCH 06/29] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E6=9B=B4=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 6 + .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/gj/data_handle/DevProcess.groovy | 16 +- .../FjwBigdataForsaleBatch.groovy | 1 + .../fangjia/script/gj/poc/AddPriceTest.groovy | 28 +++- .../gj/xiamen/GaoDePoiDataProcess.groovy | 155 +++++++++++++++++- 6 files changed, 189 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d7bf3a8a3..bcd484c2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -62,6 +62,12 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { + def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd") + def objectId = dateToObjectId(date) + println objectId + + //664a22000000000000000000 5-20 + //664b73800000000000000000 5-21 def citys = CcbCommonUtils.getRunningCitys(args) // // println "curDate:"+curDate diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 691a3f859..14c61c7e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-20") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 1098539f6..89a12c9d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -32,7 +32,7 @@ class DevProcess { // task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() -// task.matchDev() + task.matchDev() // task.calAddrDistince() // task.updateCityAndDist() @@ -44,7 +44,7 @@ class DevProcess { // task.addNewDev() // task.createDevMapping() // task.deleteAddr() - task.deleteAddr2() +// task.deleteAddr2() // task.synchDevTo43() } @@ -292,17 +292,17 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> - if(d['H']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + if(!d['F']){ def q = new BasicDBObject() - q.put("name", d['H']) + q.put("name", d['D'].replace("河北", "").replace("浙江","")) def obj = development_zone_CONN.findOne(q) if(obj){ - d['S'] = obj._id.toString() - d['T'] = obj.name + d['F'] = obj._id.toString() + d['G'] = obj.name } } - },1); + },0); } def calAddrDistince(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 2b8cde742..42a11b53a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -180,6 +180,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ m.house_structure_code = null//getHouseStructureCode(); m.source = "FJW" retMap.put(m.houseId, converData(m)) + houseIdsfilter.put(m.houseId) }else{ println city +" "+m.houseId+" 已存在" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 93a106c85..dc9e9b1c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,17 +33,31 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\苏州价格.xlsx" - addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx" +// addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() + addTradePrice(filePath) } + def addTradePrice(filePath){ + ExcelUtils.multiThreadingModify(filePath, { d -> + if(d['Y']){ + def dist = CommonDatautils.findDistrictById(d['Y']) + def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01') + if (tradePrice) { + d['W'] = tradePrice.toInteger() + } + } + + }); + } + def addRegion(){ ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d-> def dist = CommonDatautils.findDistrictById(d['A']) @@ -190,11 +204,11 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['C']) { - def dist = CommonDatautils.findDistrictById(d['C']) + if (!d['X']) { + def dist = CommonDatautils.findDistrictById(d['Y']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) - d['O'] = thirdPrice + d['X'] = thirdPrice } } }) @@ -206,8 +220,8 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) - +// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) + q.put("s_date", 202404) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index cfda2228d..6d41f48cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -39,7 +39,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) public static def version = new Date().getTime() @@ -58,6 +58,9 @@ class GaoDePoiDataProcess extends BaseBatch{ def fullCityMap = null + DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL) static main(args) { runBatch(new GaoDePoiDataProcess(), args) @@ -69,12 +72,152 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchMapPoi(); - synchCompany() +// synchCompany() // poiCatProcess() // addressAndCatProcess() // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() +// sysnchVillage() + + addTownship() + } + + def addTownship(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d-> + if(d['J']){ +// def gaodeMap = getTownshipInfo(d['F']+","+d['E']) +// if(gaodeMap && gaodeMap.township){ +// d['H'] = gaodeMap.city +// d['I'] = gaodeMap.district +// d['J'] = gaodeMap.township +// d['K'] = gaodeMap.format_addr +// } + + def q = new BasicDBObject() + q.put("city", d['H']) + q.put("district", d['I']) + q.put("street",d['G']) + def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q) + if(obj){ + d['O'] = "是" + } + } + }) + + } + + def sysnchVillage(){ + def pois = findAllVillagePoiMap() + + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(pois.size()) + + + + for(def i=0;i Date: Fri, 24 May 2024 16:30:39 +0800 Subject: [PATCH 07/29] =?UTF-8?q?=E5=8D=97=E4=BA=AC/=E5=B9=BF=E5=A4=A7?= =?UTF-8?q?=E6=B6=89=E5=86=9C=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddrSvcController.java | 7 ++----- .../com/fangjia/open/controllers/GdBankController.java | 4 ++-- .../com/fangjia/open/controllers/PropertyController.java | 7 +++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index c0f3d23ac..dbbad6549 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -435,11 +435,8 @@ public class AddrSvcController { if (json.containsKey("result")) { //我们的结果 JSONObject result = json.getJSONObject("result"); - // 如果是无效信息,则用我们的 - if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) { - basicModel.setResult(result); - }//我们省级及以上有开发区,直接用我们的结果 - else if (result.containsKey("devDistrictName")) { + //我们省级及以上有开发区,直接用我们的结果 + if (result.containsKey("devDistrictName")) { basicModel.setResult(result); } else { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java index 06d3ef504..dfa009521 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -95,8 +95,8 @@ public class GdBankController { } else if (("否").equals(result.get("countyLevelCity"))) { result.put("prefectureLevelCity", "是"); } - // 如果是无效信息,则用我们的 - if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) { + // 如果是无效信息或者没有行政区划,则用我们的 + if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) { basicModel.setResult(result); //我们省级及以上有开发区,直接用我们的结果 } else if (result.containsKey("devDistrictName")) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 875390efc..2e741bfef 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -2539,8 +2539,15 @@ public class PropertyController { districtNames.add(districtTemp.getName()); } } + + // districtNames 不包含 主小区名,加进去 + if (!districtNames.contains(masterDistrictName)){ + districtNames.add(masterDistrictName); + } + } } + // List dealPrices = new ArrayList(); // 将主名与映射别名查得成交加入 存在不能查出最新成交问题 // for (String name : districtNames) { From e5320e1183686f0d28a0e73f0050050500cd0e43 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 26 May 2024 00:05:40 +0800 Subject: [PATCH 08/29] update --- .../fangjia/script/ccbevaluation/Test.groovy | 70 +++--- .../script/gj/data_handle/DevProcess.groovy | 96 ++++++-- .../script/gj/meishubao/MsbPocTest.groovy | 52 ++--- .../fangjia/script/gj/poc/AddPriceTest.groovy | 6 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 211 ++++++++++++------ .../script/gj/xiamen/SeaweedDataSynch.groovy | 19 +- .../fangjia/script/util/ElasticUtils.groovy | 2 +- 7 files changed, 295 insertions(+), 161 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 7978ddbeb..019c22494 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,52 +1289,54 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格不作为指导价录入5-20.xlsx" -// -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['J']){ -//// def region = RegionData.formatRegion(d['B'], d['C']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") -//// if(dist){ -//// d['G'] = dist.region -//// d['H'] = dist._id.toString() -//// } -//// } -// if(!d['A']){ -// def dist = CommonDatautils.findDistrictById(d['J']) +def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx" + +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['G']){ +// def region = RegionData.formatRegion(d['B'], d['C']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // if(dist){ +// d['G'] = dist.region +// d['H'] = dist._id.toString() +// } +// } + if(!d['H'] && d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) + if(dist){ // d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat -// } -// } -// } -//},0) - -def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" -ExcelUtils.multiThreadingModify(filePath, { d -> - if(d['B']){ - - def obj = CommonDatautils.findDistrictById(d['B']) - if(obj){ - def markMap = obj.mark_map - if(!obj.mark_map){ - markMap = [:] + d['H'] = dist.name } - def data = new BasicDBObject() - markMap.put("exclusive", "是") - data.put("mark_map", markMap) - data.put("m_date", new Date()) - DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) } } +},0) -}) + +//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" +//ExcelUtils.multiThreadingModify(filePath, { d -> +// if(d['B']){ +// +// def obj = CommonDatautils.findDistrictById(d['B']) +// if(obj){ +// def markMap = obj.mark_map +// if(!obj.mark_map){ +// markMap = [:] +// } +// def data = new BasicDBObject() +// markMap.put("exclusive", "是") +// data.put("mark_map", markMap) +// data.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) +// } +// } +// +//}) //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 89a12c9d2..7c1a06a58 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,10 +29,10 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() - task.matchDev() +// task.matchDev() // task.calAddrDistince() // task.updateCityAndDist() @@ -46,6 +46,9 @@ class DevProcess { // task.deleteAddr() // task.deleteAddr2() // task.synchDevTo43() +// task.addNewDev2() + +// task.addAlias() } @@ -165,10 +168,10 @@ class DevProcess { def devSplitRoad(){ def i = 0; - while(i<1){ - def fileName = "/home/goojia/gj/company-${i}.xlsx" + while(i<14){ + def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\开发区\\56万企业地址\\省没出值地方.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\浙江\\zhejiang_0.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ @@ -194,7 +197,7 @@ class DevProcess { } } }*/ - if(d['C'] && d['B']!='河北省' && !d['J']){ + if(d['C'] && !d['J']){ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) if(jsonObject){ def street = jsonObject.get("street") @@ -292,10 +295,10 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上.xlsx",{d-> if(!d['F']){ def q = new BasicDBObject() - q.put("name", d['D'].replace("河北", "").replace("浙江","")) + q.put("name", d['C']) def obj = development_zone_CONN.findOne(q) if(obj){ d['F'] = obj._id.toString() @@ -507,6 +510,27 @@ class DevProcess { }) } + def addNewDev2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + if(d['F'] && d['F']=="新增"){ + def udata = new BasicDBObject() + udata.create_time = new Date() + udata.province = d['G'] + udata.city = d['I'] + udata.district = d['J'] + udata.level = "省级" + udata.name = d['D'] + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + } + }) + } + def createDevMapping(){ def dbcursor = development_zone_CONN.find(new BasicDBObject()) for(def d : dbcursor){ @@ -550,15 +574,15 @@ class DevProcess { } def deleteAddr2(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北开发区删除地址.xlsx",{d-> - if(d['E']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d-> + if(d['Q'] && d['M']){ def q = new BasicDBObject() - q.put("_id", new ObjectId(d['E'])) + q.put("_id", new ObjectId(d['Q'])) def obj = development_zone_CONN.findOne(q) if(obj){ def q2 = new BasicDBObject() q2.put("name", obj.name) - q2.put("address", d['A']+d['B']) + q2.put("address", d['I']+d['J']) def sameObj = DEV_MAPPING_CONN.findOne(q2) if(sameObj){ DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id)) @@ -580,17 +604,19 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万企业地址\\56万需修改地址.xlsx",{d-> - if(d['L'] && d['L']=="路正常" && d['K']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上(49个对应名称1个新增).xlsx",{d-> + if(d['G']){ def q = new BasicDBObject() - q.put("name", d['K']) + q.put("name", d['E']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['Q'] = obj._id.toString() - d['R'] = obj.name - def address = d['I']+d['J'] - q.put("address", address) + d['N'] = obj._id.toString() + d['O'] = obj.name +// def address = d['F']//d['I']+d['J'] +// q.put("address", address) + def street = d['G'] + q.put("township", street) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ def udata = new BasicDBObject() @@ -603,13 +629,39 @@ class DevProcess { } udata.put("name",obj.name) - udata.put("type", "地址") - udata.put("address", address) +// udata.put("type", "地址") + udata.put("type", "乡镇街道") + udata.put("township", street) DEV_MAPPING_CONN.save(udata) - println obj.name +" "+address + println obj.name +" "+street } } } },0); } + + def addAlias(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d-> + def q = new BasicDBObject() + q.put("name", d['A']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['D'] = obj._id.toString() + def arrs = d['B'].split("、") + def alias = obj.alias + if(!alias){ + alias = [] + } + for(def s : arrs){ + if(!alias.contains(s)){ + alias << s + } + } + + def udata = new BasicDBObject() + udata.put("alias", alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index 1c67e8706..99b913307 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -13,7 +13,7 @@ class MsbPocTest { task.run(); } def run(){ - def inputfile = "D:\\数据\\建行\\房产估值\\202311广州.xlsx" + def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" // def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx // standard(inputfile) // match(inputfile) @@ -69,21 +69,21 @@ class MsbPocTest { def matchOld(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(!row['S'] && row['D']){ + if(!row['Y'] && row['G']){ def city = row['C'] - def region = row['D'] - def name = row['F'] - def address = row['H'] + def region = null //row['D'] + def name = row['E'] + def address = row['G'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name) + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) if(jsonObject != null && jsonObject.precise!=-1) { - row['S'] = jsonObject.get("id") - row['T'] = jsonObject.get("city") - row['U'] = jsonObject.get("district") - row['V'] = jsonObject.get("name") - row['W'] = jsonObject.get("propertyType") - row['X'] = jsonObject.get("credit") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") + row['AC'] = jsonObject.get("propertyType") +// row['X'] = jsonObject.get("credit") } } }) @@ -111,18 +111,18 @@ class MsbPocTest { def match(file){ ExcelUtils.multiThreadingModify(file, { row -> - if(!row['N']){ - def city = row['A'] - def region = row['B'] - def address = row['C'] //row['G']//row['F'] + if(!row['Y']){ + def city = row['C'] + def region = null //row['B'] + def address = row['G'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) + def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false) if(jsonObject != null) { - row['N'] = jsonObject.get("id") - row['O'] = jsonObject.get("city") - row['P'] = jsonObject.get("district") - row['Q'] = jsonObject.get("name") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") } } }) @@ -169,17 +169,17 @@ class MsbPocTest { def addCCBPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> - if(!row['H']){ + if(!row['V']){ // def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") // if(dist){ - if(row['D'] instanceof String){ - def d = CommonDatautils.findDistrictById(row['D']) + if(row['Y'] instanceof String){ + def d = CommonDatautils.findDistrictById(row['Y']) if(d){ def cityId = cityCodeDataMap.get(d.city) def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId) if(map){ - row['H'] = map.avg_price - row['I'] = map.calc_type + row['V'] = map.avg_price +// row['I'] = map.calc_type } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index dc9e9b1c6..dbbf916d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,14 +33,14 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) -// addThirdPrice(filePath) + addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() - addTradePrice(filePath) +// addTradePrice(filePath) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 6d41f48cd..0682ee62c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -39,7 +41,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static def version = new Date().getTime() @@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) @@ -78,29 +80,38 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() -// sysnchVillage() + sysnchVillage() - addTownship() +// addTownship() } def addTownship(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d-> - if(d['J']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d-> + if(d['E']){ // def gaodeMap = getTownshipInfo(d['F']+","+d['E']) // if(gaodeMap && gaodeMap.township){ -// d['H'] = gaodeMap.city -// d['I'] = gaodeMap.district -// d['J'] = gaodeMap.township -// d['K'] = gaodeMap.format_addr +// if(gaodeMap.city && gaodeMap.city!="[]"){ +// d['H'] = gaodeMap.city +// } +// if( gaodeMap.district && gaodeMap.district!='[]'){ +// d['I'] = gaodeMap.district +// } +// if(gaodeMap.township && gaodeMap.township!='[]'){ +// d['J'] = gaodeMap.township +// } +// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){ +// d['K'] = gaodeMap.format_addr +// } +// // } def q = new BasicDBObject() q.put("city", d['H']) q.put("district", d['I']) - q.put("street",d['G']) + q.put("street",d['J']) def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q) if(obj){ - d['O'] = "是" + d['L'] = "是" } } }) @@ -108,107 +119,161 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def pois = findAllVillagePoiMap() + def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"] + for(def city : buducitys){ + def pois = findAllVillagePoiMap(city) + def rows = createVillageData(city, pois) + save(city, rows) + saveToMG(city, rows) + println city +" "+pois.size()+" add:"+rows.size() + } + + } + + def createVillageData(city, pois){ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(pois.size()) - - + def townMap = findTownRange(city) for(def i=0;i + if(row['Q'] && row['Q']=='是'){ + def udata = new BasicDBObject() + udata.put("township", row['J']) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata)) + } + }) + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index f14bedadd..96bbbbfcd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -92,7 +92,7 @@ class ElasticUtils { HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { httpAsyncClientBuilder.disableAuthCaching(); CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));//goojia123456 + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); return httpAsyncClientBuilder; } From a528f7ed9ff188ef7c0a31ef1b24f5f7d47a5016 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 26 May 2024 16:36:35 +0800 Subject: [PATCH 09/29] =?UTF-8?q?=E6=88=BF=E6=BA=90=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DealAndThirdPriceDataProcess.groovy | 2 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../script/gj/data_handle/DevProcess.groovy | 18 +++++++++--------- .../FjwBigdataForsaleBatch.groovy | 5 +++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 023cc157d..5072828fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -389,7 +389,7 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" // def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all - def citys = CcbCommonUtils.citys_all + def citys = ["上海","广州"]//CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index b0756a439..4e70b43c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch { // } // } // - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 7c1a06a58..5f59e63b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,10 +29,10 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() -// task.matchDev() + task.matchDev() // task.calAddrDistince() // task.updateCityAndDist() @@ -168,10 +168,10 @@ class DevProcess { def devSplitRoad(){ def i = 0; - while(i<14){ + while(i<1){ def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\开发区\\浙江\\zhejiang_0.xlsx" + fileName = "D:\\数据\\数据建设\\浙江-湖州.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ @@ -295,14 +295,14 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上.xlsx",{d-> - if(!d['F']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万).xlsx",{d-> + if(!d['N']){ def q = new BasicDBObject() - q.put("name", d['C']) + q.put("name", d['L']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['F'] = obj._id.toString() - d['G'] = obj.name + d['N'] = obj._id.toString() + d['O'] = obj.name } } },0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 42a11b53a..f9352149c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -133,7 +133,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ println city +" 小区量:"+districts.size() - + BloomFilter curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); def retMap = new ConcurrentHashMap() for(def i=0;i Date: Sun, 26 May 2024 17:05:32 +0800 Subject: [PATCH 10/29] =?UTF-8?q?=E5=AF=86=E7=A0=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 96bbbbfcd..f14bedadd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -92,7 +92,7 @@ class ElasticUtils { HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { httpAsyncClientBuilder.disableAuthCaching(); CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));//goojia123456 httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); return httpAsyncClientBuilder; } From 1dfd7ed919d598201402412b3cd23ddd0b82fa6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 27 May 2024 14:21:42 +0800 Subject: [PATCH 11/29] =?UTF-8?q?=E6=90=9C=E7=B4=A2=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=88=AB=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 22 ++++++++++++++++++- .../com/fangjia/open/dto/PropertyDto.java | 12 ++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 2e741bfef..f22a692fa 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -39,6 +39,7 @@ import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.open.dto.addressGovernance.StructureDto; import com.fangjia.open.util.CommonDtoUtils; +import com.fangjia.service.address.AddressParserUtil; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; @@ -186,7 +187,7 @@ public class PropertyController { private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; - private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" }; + private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" }; static final int BAI_DU_GEO = 1;// 百度系坐标 static final int GAO_DE_GEO = 2;// 高德系坐标 @@ -609,6 +610,25 @@ public class PropertyController { dto.setBlock(district.getBlock()); } dto.setPropertyType(district.getEstateType2()); + if (CollectionUtils.isNotEmpty(district.getAlias())) { + // 将最相近的别名输出给用户 + String alias = null; + double similarity = 0; + if (StringUtils.isNotBlank(model.getKeyword())) { + for (String mappingAlias : district.getAlias()) { + double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword()); + if (curr > similarity ) { + similarity = curr; + alias = mappingAlias; + } + } + } + if (StringUtils.isNotBlank(alias)) { + dto.setAlias(alias); + } else { + dto.setAlias(district.getAlias().get(0)); + } + } list.add(dto); } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java index 8e9097e2a..aac0f83b2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java @@ -15,6 +15,10 @@ public class PropertyDto { private String propertyType; private String block; private String distance; + /** + * 新增小区别名 + */ + private String alias; public String getId() { return id; } @@ -88,4 +92,12 @@ public class PropertyDto { public void setDistance(String distance) { this.distance = distance; } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } } From 7c1b18afa84d6316a0ba9b94c1b421bfa759ff7c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 May 2024 14:51:49 +0800 Subject: [PATCH 12/29] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../script/gj/data_handle/DevProcess.groovy | 41 ++++++++++--------- .../FjwThirdPartyPriceBatch.groovy | 9 ++-- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 343189b24..967d6959a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -15,8 +15,8 @@ import groovy.sql.Sql class CcbCommonUtils { - //"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都", - public static def citys_fjw_all = [ "沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + //"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照", + public static def citys_fjw_all = [ "武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] //"乌鲁木齐", public static def citys_all = ["上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 5f59e63b5..e32e9a79f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -32,13 +32,13 @@ class DevProcess { // task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() - task.matchDev() +// task.matchDev() // task.calAddrDistince() // task.updateCityAndDist() // task.addDevAddres1() //按添加地址中包含开发区名称的地址 // task.addDevAddres2()//添加相差2公里范围内的地址 -// task.matchDevAndAddAddress() + task.matchDevAndAddAddress() // task.stat() // task.addNewDev() @@ -295,14 +295,14 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万).xlsx",{d-> - if(!d['N']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d-> + if(!d['P'] && d['R']){ def q = new BasicDBObject() - q.put("name", d['L']) + q.put("name", d['R']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['N'] = obj._id.toString() - d['O'] = obj.name + d['P'] = obj._id.toString() + d['Q'] = obj.name } } },0); @@ -604,19 +604,19 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上(49个对应名称1个新增).xlsx",{d-> - if(d['G']){ + ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d-> + if(d['R']){ def q = new BasicDBObject() - q.put("name", d['E']) + q.put("name", d['R']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['N'] = obj._id.toString() - d['O'] = obj.name -// def address = d['F']//d['I']+d['J'] -// q.put("address", address) - def street = d['G'] - q.put("township", street) + d['P'] = obj._id.toString() + d['Q'] = obj.name + def address = d['L']+d['M'] + q.put("address", address) +// def street = d['G'] +// q.put("township", street) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ def udata = new BasicDBObject() @@ -629,11 +629,12 @@ class DevProcess { } udata.put("name",obj.name) -// udata.put("type", "地址") - udata.put("type", "乡镇街道") - udata.put("township", street) + udata.put("type", "地址") + udata.put("address",address) +// udata.put("type", "乡镇街道") +// udata.put("township", street) DEV_MAPPING_CONN.save(udata) - println obj.name +" "+street + println obj.name +" "+address } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index 13b557f07..07d8e5679 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -53,10 +53,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch { def stop = false def month = -1 while(!stop){ - - def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) - println weekend.toString() +" "+sDate - runDatas(citys) + if(sDate<202110){ + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + println weekend.toString() +" "+sDate + runDatas(citys) + } month = month -1 sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM")); if(sDate == 201912){ From f4039dae161cad06147c659ca7ab74981cc259a5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 May 2024 20:13:50 +0800 Subject: [PATCH 13/29] =?UTF-8?q?=E6=96=B0=E8=A1=8C=E6=83=85=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCitypriceBatch.groovy | 143 +++++++++++++----- 1 file changed, 107 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 479813af4..cbb308586 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{ def size = regionDatas.size() final CountDownLatch latch = new CountDownLatch(size) def totalCount = new AtomicLong() + for(def i = 0;i3){ - sell_avg_price = avgValue(sellMap.avg_price) + if(sellMap){ sell_avg_area = avgValue(sellMap.area) - sell_avg_total = avgValue(sellMap.total_price) - }else{ - sell_count = 0 - sellDataType = null + sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price) +// if(sell_count>3){ +// if(allCcbPriceMap.containsKey(dist._id.toString())){ +// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price) +// sell_avg_area = avgValue(sellMap.area) +// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price) +// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString()) +// }else{ +// sell_count = 0 +// sellDataType = null +// } +// }else{ +// sell_count = 0 +// sellDataType = null +// } } } + def rent_avg_price = null def rent_avg_area = null def rent_avg_total = null @@ -381,6 +412,8 @@ class BigdataCitypriceBatch extends BaseBatch{ rentDataType = null } } + + //三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格 if(!sell_avg_price){ sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) } @@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{ def sell_avg_area = null def sell_avg_total = null def sellStartDate = null + + def avgPrice = null + if(region){ + avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6)) + }else{ + avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6)) + } + + if (sell_count < 10) { sellDataType = 3 sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); @@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{ sellStartDate = startDate } if (sell_count) { - sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") + sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area") sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price") } @@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{ return datas; } + def findCommunityPriceByCityMap(cityCode, month){ + def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def dataMap = [:] + pgsqlInstance.eachRow(sql, { r -> + dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc) + }) + return dataMap; + } + + def findCommunityPriceCountByCityMap(cityCode, month){ + def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}"; + def dataMap = [:] + pgsqlInstance.eachRow(sql, { r -> + if(r.gujiabdsrt3mlgcasenum){ + dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum) + } + }) + return dataMap; + } + def findCcbCityPrice(cityCode, month){ def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; def avgPrice = null @@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{ return regionPriceMap; } + def findCcbRegionPriceByRegion(cityCode, regionCode, month){ + def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'"; + def price = null + pgsqlInstance.eachRow(sql, { r -> + price = r.gujia_bds_clc_avprc + }) + return price; + } + def districtCityPrictRate(cityPrice, datas){ def distCityRateMap = [:] From f5579a2a699b818d8f7d643d1de099488e94fd9a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 May 2024 21:13:07 +0800 Subject: [PATCH 14/29] bug --- .../ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index cbb308586..535748054 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -414,9 +414,9 @@ class BigdataCitypriceBatch extends BaseBatch{ } //三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格 - if(!sell_avg_price){ - sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) - } +// if(!sell_avg_price){ +// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) +// } if(!rent_avg_price){ rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) } From 9b225bd3b365e3c554edfdd28266432ab3f9912d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 30 May 2024 15:50:58 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E6=B2=B3=E5=8C=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/data_handle/DevProcess.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index e32e9a79f..52b16261b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,7 +29,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -38,7 +38,7 @@ class DevProcess { // task.updateCityAndDist() // task.addDevAddres1() //按添加地址中包含开发区名称的地址 // task.addDevAddres2()//添加相差2公里范围内的地址 - task.matchDevAndAddAddress() +// task.matchDevAndAddAddress() // task.stat() // task.addNewDev() @@ -169,7 +169,7 @@ class DevProcess { def devSplitRoad(){ def i = 0; while(i<1){ - def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx" + def fileName = "/home/goojia/gj/hebei-${i}.xlsx" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\数据建设\\浙江-湖州.xlsx" } From fc15aa319758b99873765b4f7fc6f8c8f8bde9d5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 3 Jun 2024 18:31:30 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E6=94=B9bug,=E5=B0=91=E4=BA=86=E4=B9=8C?= =?UTF-8?q?=E9=B2=81=E6=9C=A8=E9=BD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 967d6959a..f3f22c296 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -15,10 +15,10 @@ import groovy.sql.Sql class CcbCommonUtils { - //"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照", - public static def citys_fjw_all = [ "武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] - //"乌鲁木齐", - public static def citys_all = ["上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] + //"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", + public static def citys_fjw_all = [ "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + + public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] From 793736676425999393cd94709b20ef63ae4650b1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 4 Jun 2024 10:02:26 +0800 Subject: [PATCH 17/29] =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E5=8C=BA=E5=8A=A0=E8=A1=97=E9=81=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/DevProcess.groovy | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 52b16261b..13d1b5009 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -49,6 +51,7 @@ class DevProcess { // task.addNewDev2() // task.addAlias() +// task.addDevShape()//添加开发区围栏 } @@ -171,7 +174,7 @@ class DevProcess { while(i<1){ def fileName = "/home/goojia/gj/hebei-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\浙江-湖州.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\江苏-无锡.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ @@ -197,32 +200,48 @@ class DevProcess { } } }*/ - if(d['C'] && !d['J']){ + if(d['C'] && !d['G']){ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) if(jsonObject){ def street = jsonObject.get("street") def streetNumber = jsonObject.get("streetNumber") if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['J'] = jsonObject.get("province") + d['G'] = jsonObject.get("province") } if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['K'] = jsonObject.get("city") + d['H'] = jsonObject.get("city") } if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['L'] = jsonObject.get("district") + d['I'] = jsonObject.get("district") + } + + if(jsonObject.get("township") && jsonObject.get("township")!="null"){ + d['J'] = jsonObject.get("township") } if(jsonObject && street && streetNumber && street!="null"){ - d['M'] = jsonObject.get("street") + d['K'] = jsonObject.get("street") if(streetNumber!='null'){ - d['N'] = jsonObject.get("streetNumber") + d['L'] = jsonObject.get("streetNumber") + + def road = jsonObject.get("street")+jsonObject.get("streetNumber") + if(!d['C'].contains(road)){ + d['N'] = "原地址不含路" + } + + if(!d['N']){ + def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$') + if(!flag){ + d['N'] = "路非法" + } + } } } if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ - d['O'] = jsonObject.get("dev") + d['M'] = jsonObject.get("dev") } } @@ -665,4 +684,18 @@ class DevProcess { } }) } + + def addDevShape(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d-> + if(d['W']){ + def s = d['S'] + JSONObject jsonObject = JSONObject.fromObject(s); + Map map = DataHandleUtils.JsonToMap(jsonObject); + + def udata = new BasicDBObject() + udata.put("shape", map) + development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata)) + } + }) + } } From 8f3cad2cfde215b497ce51ffdb7f94cef3b55e49 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Thu, 6 Jun 2024 10:40:16 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E5=8C=97=E4=BA=A4=E6=89=80=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=8F=82=E6=95=B0preciseLevel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/ComprehensiveQueryController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java index 55872ef9d..9d891b609 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java @@ -235,6 +235,7 @@ public class ComprehensiveQueryController { communityAvgPriceParams.put("category", model.getCategory()); // communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); communityAvgPriceParams.put("suggestName", String.valueOf(model.getName())); + communityAvgPriceParams.put("preciseLevel", "strict"); communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"); JSONObject matchResponse = null; try { From b925bd17d41d07b5362429596a833f05ca90e42e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 Jun 2024 11:51:25 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8D=97=E4=BA=AC?= =?UTF-8?q?=E9=93=B6=E8=A1=8C=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddrSvcController.java | 95 ++++++++++++++++++- .../open/controllers/GdBankController.java | 2 +- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index dbbad6549..5f9b37bb7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -391,13 +391,106 @@ public class AddrSvcController { } }; - /** * 涉农识别-新规则-南京银行 * * @param model * @return */ + public Object njcbVillageIdentifyAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); + String address = model.getAddress(); + params.put("address", address); + String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null; + params.put("type", type); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8"); + } catch (Exception e) { + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("msgCode"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if (json.containsKey("result")) { + //我们的结果 + JSONObject result = json.getJSONObject("result"); + //我们省级及以上有开发区,直接用我们的结果 + if (result.containsKey("devDistrictName")) { + basicModel.setResult(result); + + } else { + try { + //如果是江苏省或者浙江省 调用齐梁的结果 + if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) { + //结果比较,如果涉农结果不一致 + //调用齐梁 type :1 对公 0 对私 + //我们 type :1 对私 2 对共 + String qltype = "2".equals(type) ? "1" : "0"; + Map map = senRequest(address, qltype); + // 涉农结果不一样 + if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) { + //城市不一样 用我们 + if (result.containsKey("city")) { + if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) { + basicModel.setResult(result); + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + + } else { + basicModel.setResult(result); + } + + } else { + basicModel.setResult(result); + + } + + } catch (Exception e) { + e.printStackTrace(); + basicModel.setResult(result); + } + + } + + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + + /** + * 涉农识别-新规则-基础版本 + * + * @param model + * @return + */ public Object villageIdentifyAction(AddressModel model) { RestfulModelAndView mav = new RestfulModelAndView(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java index dfa009521..fd1709bf3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -51,7 +51,7 @@ public class GdBankController { }; /** - * 涉农识别-新规 + * 涉农识别-新规则-光大银行 * * @param model * @return From 191f52dbfbdea7ad2f363c562ef70a02b60cf08a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 Jun 2024 16:37:44 +0800 Subject: [PATCH 20/29] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=90=9C=E6=88=BF?= =?UTF-8?q?=E5=88=AB=E5=90=8D=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/dal/mongo/dao/AliasFangDao.java | 16 +++++ .../dal/mongo/dao/AliasFangDaoImpl.java | 40 +++++++++++ .../fangjia/dal/mongo/model/AliasFang.java | 68 +++++++++++++++++++ .../open/controllers/PropertyController.java | 40 +++++++---- .../service/datastore/DistrictService.java | 1 + .../datastore/DistrictServiceImpl.java | 1 - .../service/open/AliasFangService.java | 23 +++++++ .../service/open/AliasFangServiceImpl.java | 27 ++++++++ 8 files changed, 201 insertions(+), 15 deletions(-) create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java new file mode 100644 index 000000000..004f28bb8 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java @@ -0,0 +1,16 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public interface AliasFangDao extends MongoDao { + + List findFangByNameAndAlias(String city, String region, Set names); +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java new file mode 100644 index 000000000..e5fe59202 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java @@ -0,0 +1,40 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.model.AliasFang; +import com.fangjia.dal.mongo.model.ConsumerAddress; +import com.mongodb.BasicDBList; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import org.apache.commons.lang.StringUtils; +import org.sothis.core.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public class AliasFangDaoImpl extends MongoDaoImpl implements AliasFangDao{ + + @Override + public List findFangByNameAndAlias(String city, String region, Set names) { + DBObject query = new BasicDBObject(); + if (StringUtils.isNotBlank(city)) { + query.put("city", city); + } + if (StringUtils.isNotBlank(region)) { + query.put("region", region); + } + // db.alias_fang.find({'city': '重庆', 'region': '渝北', $or: [{'name': '中国摩'}, {'alias': '中国摩'}]}) + if(CollectionUtils.isNotEmpty(names)) { + BasicDBList dbList = new BasicDBList(); + dbList.add(new BasicDBObject("name", new BasicDBObject("$in", names))); + dbList.add(new BasicDBObject("alias", new BasicDBObject("$in", names))); + query.put("$or", dbList); + } + List objs = find(query, null, null, 0, Integer.MAX_VALUE); + return objs; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java new file mode 100644 index 000000000..c4027ecc3 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java @@ -0,0 +1,68 @@ +package com.fangjia.dal.mongo.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; + +/** + * 搜房名称与别名关系 + */ +@Entity +@Table(name = "business.alias_fang") +public class AliasFang implements MongoEntity { + + private String id; + private String city; + private String region; + private String name; + private String alias; + + @Override + @Id + @GeneratedValue + @Column(name = "_id", unique = true, nullable = false) + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "alias") + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index f22a692fa..b5efca725 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -10,17 +10,7 @@ import java.net.URLEncoder; import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,6 +20,7 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; +import com.fangjia.dal.mongo.dao.AliasFangDao; import com.fangjia.dal.mongo.model.*; import com.fangjia.dal.mongo.open.model.CommunityBasePrice; import com.fangjia.dal.mongo.open.model.CommunityPoi; @@ -139,7 +130,6 @@ import com.mongodb.DBObject; @Authority(role = Role.USER) public class PropertyController { - private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class); private DistrictService districtService; private AddressParser addressParser; @@ -197,6 +187,7 @@ public class PropertyController { private DistrictRoomSqmetreRentService districtRoomSqmetreRentService; private RentDealPriceService rentDealPriceService; private EvaluateRatioService evaluateRatioService; + private AliasFangService aliasFangService; private static final int EVALUATE_SHOW_DETAIL = 1; private static final Map PROPERTY_TYPES = new HashMap() { @@ -2536,7 +2527,7 @@ public class PropertyController { model.setMax(100); } - List districtNames = new ArrayList(); + Set districtNames = new HashSet(); if (null == district) { districtNames.add(model.getName()); } else { @@ -2567,6 +2558,23 @@ public class PropertyController { } } + // 将搜房中的别名与名称加入查询集合 + try { + List aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames); + Set aliasSet = new HashSet(); + for (AliasFang aliasFang : aliasFangs) { + if (StringUtils.isNotBlank(aliasFang.getName())) { + aliasSet.add(aliasFang.getName()); + } + if (StringUtils.isNotBlank(aliasFang.getAlias())) { + aliasSet.add(aliasFang.getAlias()); + } + } + districtNames.addAll(aliasSet); + } catch (Exception ex) { + ex.printStackTrace(); + } + // List dealPrices = new ArrayList(); // 将主名与映射别名查得成交加入 存在不能查出最新成交问题 @@ -2576,7 +2584,7 @@ public class PropertyController { // dealPrices.addAll(dealPricesTempList); // } // } - List dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), districtNames, model.getMax() * 5, model); + List dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList(districtNames), model.getMax() * 5, model); if (model.getSize() != null) { Collections.sort(dealPrices, new Comparator() { @Override @@ -6025,4 +6033,8 @@ public class PropertyController { public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { this.communityBasePriceService = communityBasePriceService; } + + public void setAliasFangService(AliasFangService aliasFangService) { + this.aliasFangService = aliasFangService; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java index 68588d8e5..04e5d5de2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java @@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery; import com.fangjia.dal.SortParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank; +import com.fangjia.dal.mongo.model.AliasFang; import com.fangjia.dal.mongo.model.DistrictFacilitys; import com.fangjia.dal.seaweed.model.District; import com.fangjia.service.bo.DistrictInfo; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index 189508ed8..dd625b469 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService { params.add(new QueryParam("$sr", new SrQuery(query))); return districtDao.findAllByParams(params); } - } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java new file mode 100644 index 000000000..c3136da96 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java @@ -0,0 +1,23 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public interface AliasFangService { + + /** + * 查询搜房别名与名称关系 + * + * @param city + * @param region + * @param names + * @return + */ + List findFangByNameAndAlias(String city, String region, Set names); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java new file mode 100644 index 000000000..2437795cd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java @@ -0,0 +1,27 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.dao.AliasFangDao; +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public class AliasFangServiceImpl implements AliasFangService{ + + private AliasFangDao aliasFangDao; + + @Override + public List findFangByNameAndAlias(String city, String region, Set names) { + return aliasFangDao.findFangByNameAndAlias(city, region, names); + } + + public void setAliasFangDao(AliasFangDao aliasFangDao) { + this.aliasFangDao = aliasFangDao; + } +} From 771ee6af5de0aacc22138731492078ba48de9750 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 13 Jun 2024 10:54:55 +0800 Subject: [PATCH 21/29] =?UTF-8?q?=E6=94=B9=E6=96=87=E4=BB=B6=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 75 ++++++++++++++++++- .../script/gj/data_handle/DevProcess.groovy | 41 +++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ea417b329..634bf7b80 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -18,6 +18,7 @@ import groovy.json.JsonSlurper import net.sf.json.JSONArray import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils +import org.apache.poi.ss.util.RegionUtil import org.bson.Document import org.bson.types.ObjectId @@ -117,6 +118,7 @@ class CommonDatautils { } public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009") + public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -274,7 +276,7 @@ class CommonDatautils { q.put("status", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","泰瑞家园") + q.put("name","上唐山水") DBObject keys = new BasicDBObject(); if(fields){ @@ -4605,6 +4607,77 @@ class CommonDatautils { } + static def queryFullNameByCity(){ + def q = new BasicDBObject() + q.put("type","城市") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + def cityMap = [:] + for(def d : dbcursor){ + def city = RegionData.formatCity(d.city) + if(city){ + cityMap.put(city, d) + }else{ + //println d.city +" 格式化不出" + } + + } + return cityMap + } + + static def queryDistNameByCity(){ + + def q = new BasicDBObject() + q.put("type","区域") + def keys = new BasicDBObject() + keys.put("province",1) + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + keys.put("mzb_district_name", 1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000) + def regionMap = [:] + for(def d : dbcursor){ + if(d.mzb_district_name){ + def regions = regionMap.get(d.city) + if(!regions){ + regions = [] + } + regions << d + regionMap.put(d.city, regions) + }else{ + println d.province+" "+d.district +" 格式化不出民政部区域" + } + } + return regionMap + } + + static def queryCommunityByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + + def keys = new BasicDBObject() + keys.put("province",1) + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + keys.put("shape",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.district + def list = townMap.get(key) + if(!list){ + list = [] + } + list << d + townMap.put(key, list) + } + return townMap + + } + + static def queryDistNameCity(){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 13d1b5009..c722603c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -52,6 +52,7 @@ class DevProcess { // task.addAlias() // task.addDevShape()//添加开发区围栏 +// task.matchDevTownship()//添加乡镇街道 } @@ -172,7 +173,7 @@ class DevProcess { def devSplitRoad(){ def i = 0; while(i<1){ - def fileName = "/home/goojia/gj/hebei-${i}.xlsx" + def fileName = "/home/goojia/gj/qzd-${i}.xlsx" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\数据建设\\开发区\\江苏-无锡.xlsx" } @@ -660,6 +661,44 @@ class DevProcess { },0); } + + + def matchDevTownship(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> + if(d['C']){ + def q = new BasicDBObject() + q.put("name", d['C']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['E'] = obj._id.toString() + def street = d['D'] + q.put("township", street) + def sameObj = DEV_MAPPING_CONN.findOne(q) + if(!sameObj){ + def udata = new BasicDBObject() + udata.put("province", obj.province) + if(obj.city){ + udata.put("city", obj.city) + } + if(obj.district){ + udata.put("district", obj.district) + } + + udata.put("name",obj.name) +// udata.put("type", "乡镇街道") +// udata.put("township", street) + + udata.put("type", "区域") + udata.put("dev_district", street) + udata.put("c_date",new Date()) + DEV_MAPPING_CONN.save(udata) + println obj.name +" "+street + } + } + } + },1); + } + def addAlias(){ ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d-> def q = new BasicDBObject() From 2c9fa258a2d321d0c144aa0e8744241b993cd31c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 13 Jun 2024 10:58:19 +0800 Subject: [PATCH 22/29] =?UTF-8?q?19-=E3=80=8B93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/district/FangjiaClientUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 71132614c..0bdd0555f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -354,7 +354,7 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" + def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } From b853212a451ca5addddeb19d7e7e2c52b3facd1d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 13 Jun 2024 17:55:06 +0800 Subject: [PATCH 23/29] =?UTF-8?q?=E5=AF=BC=E6=88=BF=E6=BA=90=E5=8A=A0?= =?UTF-8?q?=E5=85=A5=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 6 ++-- .../FjwBigdataForsaleBatch.groovy | 36 +++++++++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index f3f22c296..aa9eca228 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -15,8 +15,8 @@ import groovy.sql.Sql class CcbCommonUtils { - //"乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", - public static def citys_fjw_all = [ "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + + public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] @@ -440,7 +440,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","新泾三村") +// q.put("name","中爱花园") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index f9352149c..ca6ce7fd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -74,6 +74,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"); + // 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + static main(args) { runBatch(new FjwBigdataForsaleBatch(), args); } @@ -91,7 +94,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def endDate = curMonth+"19" println "房源时间:"+startDate+" "+endDate - def houseIdsfilter = findAllHouseIdsByCity() + def houseIdsfilter = findAllHouseIdsByCity() for(def city : citys){ index++ @@ -194,7 +197,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size() } catch (Exception e) { - e.printStackTrace() + println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage() } finally { latch.countDown() } @@ -270,7 +273,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){ def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) - def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) + def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) // if(rets1){ // def newRets = [] @@ -709,6 +712,33 @@ class FjwBigdataForsaleBatch extends BaseBatch{ return rets; } + + + def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) { + int retryCount = 0; + boolean success = false; + def rets = [] + + while (retryCount < MAX_TRY_COUNT && !success) { + try { + rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) + // 如果操作成功,设置success为true + success = true; + } catch (Exception e) { + retryCount++; + println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT + // 这里可以选择在重试之前等待一段时间 + Thread.sleep(3000); // 例如等待1秒 + } + } + + if (!success) { + println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name + } + return rets; + } + + /** * 是否是别墅 * @return From 4d2c5566c4b24c045bdbb4e0eee4745424b422ed Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 14 Jun 2024 11:03:14 +0800 Subject: [PATCH 24/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=AA=E6=97=B6?= =?UTF-8?q?=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index ca6ce7fd7..50f070a3a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -72,7 +72,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); - def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"); + def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); // 最大重试次数 private static final int MAX_TRY_COUNT = 5; @@ -89,9 +89,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def startDate = lastMonth+"18" - def endDate = curMonth+"19" + def endDate = curMonth+"19" //正式运行时curMonth要改 println "房源时间:"+startDate+" "+endDate def houseIdsfilter = findAllHouseIdsByCity() From 337181133b6574651d7e1cdf1b3efbd8d682ec2b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 18 Jun 2024 10:35:02 +0800 Subject: [PATCH 25/29] =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E8=B7=91?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/DevProcess.groovy | 183 +++++++++++------- 1 file changed, 110 insertions(+), 73 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index c722603c3..d74420dde 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -26,6 +26,7 @@ class DevProcess { DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") static main(args) { def task = new DevProcess(); @@ -53,6 +54,8 @@ class DevProcess { // task.addAlias() // task.addDevShape()//添加开发区围栏 // task.matchDevTownship()//添加乡镇街道 + +// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中 } @@ -170,85 +173,99 @@ class DevProcess { } } + def findDev(city, name){ + def q = new BasicDBObject() + q.put("name", name) + q.put("city", city) + def obj = development_zone_CONN.findOne(q) + return obj + } + def devSplitRoad(){ def i = 0; - while(i<1){ + while(i<20){ def fileName = "/home/goojia/gj/qzd-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\开发区\\江苏-无锡.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx" } - ExcelUtils.multiThreadingModify(fileName,{d-> - /*if(d['C'] && !d['V']){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['V'] = jsonObject.get("province") - } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['W'] = jsonObject.get("city") - } - - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['X'] = jsonObject.get("district") - } - - if(jsonObject && street && streetNumber && street!="null"){ - d['Y'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['Z'] = jsonObject.get("streetNumber") - } - } - }*/ - if(d['C'] && !d['G']){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) - if(jsonObject){ - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['G'] = jsonObject.get("province") - } - - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['H'] = jsonObject.get("city") - } - - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['I'] = jsonObject.get("district") - } - - if(jsonObject.get("township") && jsonObject.get("township")!="null"){ - d['J'] = jsonObject.get("township") - } - - if(jsonObject && street && streetNumber && street!="null"){ - d['K'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['L'] = jsonObject.get("streetNumber") - - def road = jsonObject.get("street")+jsonObject.get("streetNumber") - if(!d['C'].contains(road)){ - d['N'] = "原地址不含路" - } - - if(!d['N']){ - def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$') - if(!flag){ - d['N'] = "路非法" - } + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['E'] && !d['G']){ + def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E']) + if(jsonObject1) { + if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") { + def devName = jsonObject1.get("dev") + def obj = findDev(d['B'], devName) + if (obj) { + d['Q'] = obj._id.toString() + d['R'] = obj.name } } } - - if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ - d['M'] = jsonObject.get("dev") - } } - } - },0) - i++ + if(d['C'] && !d['S']) { + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if (jsonObject.get("province") && jsonObject.get("province") != "null") { + d['G'] = jsonObject.get("province") + } + + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + d['H'] = jsonObject.get("city") + } + + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + d['I'] = jsonObject.get("district") + } + + if (jsonObject.get("township") && jsonObject.get("township") != "null") { + d['J'] = jsonObject.get("township") + } + + if (jsonObject && street && streetNumber && street != "null") { + d['K'] = jsonObject.get("street") + def road = jsonObject.get("street") + if (streetNumber != 'null') { + d['L'] = jsonObject.get("streetNumber") + road = road + jsonObject.get("streetNumber") + } else { + d['N'] = "信息不完整" + } + + if (!d['N'] && !d['C'].contains(road)) { + d['N'] = "原地址不含路" + } + + if (!d['N']) { + def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$') + if (!flag) { + d['N'] = "路非法" + } + def roadsize = jsonObject.get("street").size() + if (!d['N'] && (roadsize < 3 || roadsize > 7)) { + d['N'] = "路太长|路太短" + } + } + }else{ + d['N'] = "无地址" + } + + if (jsonObject.get("dev") && jsonObject.get("dev") != "null") { + def devName = jsonObject.get("dev") + def obj = findDev(d['B'], devName) + if (obj) { + d['S'] = obj._id.toString() + d['T'] = obj.name + } + } + } + },0) + i++ + } } } @@ -315,14 +332,15 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d-> - if(!d['P'] && d['R']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d-> + if(!d['Q'] && d['L']){ def q = new BasicDBObject() - q.put("name", d['R']) + q.put("name", d['L']) + q.put("city",d['G']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['P'] = obj._id.toString() - d['Q'] = obj.name + d['Q'] = obj._id.toString() + d['R'] = obj.name } } },0); @@ -737,4 +755,23 @@ class DevProcess { } }) } + + def addDevStreet(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> + if(d['J']=="是"){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道")) + for(def obj : dbcursor){ + def o = new BasicDBObject() + o.put("province" , obj.province) + o.put("city" , obj.city) + o.put("district" , obj.mzb_district_name) + o.put("name" , d['E']) + o.put("type" , "乡镇街道") + o.put("township" , obj.street) + DEV_MAPPING_CONN.save(o) + } + } + }) + + } } From b88bf9150d10bd79dd37957c808870e0980f7290 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 18 Jun 2024 13:27:14 +0800 Subject: [PATCH 26/29] =?UTF-8?q?=E8=BE=93=E5=87=BA=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index d74420dde..2003b4c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -198,6 +198,7 @@ class DevProcess { if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") { def devName = jsonObject1.get("dev") def obj = findDev(d['B'], devName) + d['F'] = devName if (obj) { d['Q'] = obj._id.toString() d['R'] = obj.name @@ -257,6 +258,7 @@ class DevProcess { if (jsonObject.get("dev") && jsonObject.get("dev") != "null") { def devName = jsonObject.get("dev") def obj = findDev(d['B'], devName) + d['M'] = devName if (obj) { d['S'] = obj._id.toString() d['T'] = obj.name From a2d6a4b3e7e0048514e9bcafbee7c57c0d523bfd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 18 Jun 2024 19:18:13 +0800 Subject: [PATCH 27/29] update --- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 16 +- .../script/commData/CommonDatautils.groovy | 106 +++++- .../script/gj/data_handle/PoiLocation.groovy | 312 +++++++++++++++++ .../fangjia/script/gj/data_handle/Test.groovy | 105 ++++++ .../gj/data_handle/TownMiddleProcess.groovy | 131 ++++++- .../data_handle/TownshipStrandUpdate.groovy | 324 ++++++++++++++++++ .../gj/data_handle/TradeDatasDistBatch.groovy | 35 +- .../FjwThirdPartyPriceBatch.groovy | 2 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 20 +- .../script/gj/village/QiliangTest.groovy | 2 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 8 +- .../script/looplineinfo/InsidePolygon.groovy | 17 + .../fangjia/script/util/ElasticUtils.groovy | 25 +- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- .../yinjihaun/amap/GaoDeApiUtils.groovy | 2 +- 17 files changed, 1078 insertions(+), 33 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index aa9eca228..7a4062270 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -440,7 +440,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","中爱花园") +// q.put("name","鸿运大院") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 14c61c7e6..c43a791b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 019c22494..d527b08dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['G']){ + if(d['J']){ // def region = RegionData.formatRegion(d['B'], d['C']) // if(region){ // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") @@ -1301,17 +1301,17 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['H'] && d['G'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['G']) + if(!d['A'] && d['J'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ -// d['A'] = dist.city -// d['B'] = dist.region -// d['C'] = dist.name + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat - d['H'] = dist.name +// d['H'] = dist.name } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 634bf7b80..e9fa92613 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -13,6 +13,7 @@ import com.mongodb.AggregationOutput import com.mongodb.client.MongoCursor import com.mongodb.client.model.Accumulators import com.mongodb.client.model.Aggregates +import com.rabbitmq.client.impl.AMQImpl import groovy.json.JsonSlurper import net.sf.json.JSONArray @@ -276,7 +277,7 @@ class CommonDatautils { q.put("status", 0); q.put("cat", cat); q.put("city", city); - q.put("name","上唐山水") +// q.put("name","托斯卡纳庄园") DBObject keys = new BasicDBObject(); if(fields){ @@ -285,7 +286,7 @@ class CommonDatautils { } } - def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) + def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000) List rets = [] for(def obj : cursor){ rets << obj @@ -4573,6 +4574,28 @@ class CommonDatautils { return townMap } + static def findAllTownRanges(){ + def q = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("town", 1) + def dbcursor = TOWN_POLYLINECOLL.find(q, keys) + def townMap = [:] + for(def d : dbcursor){ + def lists = townMap.get(d.city) + if(!lists){ + lists = [] + } + def t = [:] + t.region = d.region + t.town = d.town + lists << t + townMap.put(d.city, lists) + } + return townMap + } + static def queryFullNameCity(){ def q = new BasicDBObject() def dbcursor = administrative_division_100_CONN.find(q) @@ -4634,6 +4657,7 @@ class CommonDatautils { keys.put("district",1) keys.put("street",1) keys.put("mzb_district_name", 1) + keys.put("shape", 1) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000) def regionMap = [:] for(def d : dbcursor){ @@ -4655,6 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") + q.put("province",new BasicDBObject('$in', ['福建省','重庆市'])) def keys = new BasicDBObject() keys.put("province",1) @@ -4662,6 +4687,7 @@ class CommonDatautils { keys.put("district",1) keys.put("street",1) keys.put("shape",1) + keys.put("alias",1) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) def townMap = [:] for(def d : dbcursor){ @@ -4678,6 +4704,34 @@ class CommonDatautils { } + static def queryNewTownByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city + def list = townMap.get(key) + if(!list){ + list = [] + } + def t = [:] + t.region = d.district + t.town = d.street + list << t + townMap.put(key, list) + } + return townMap + + } + + static def queryDistNameCity(){ def q = new BasicDBObject() @@ -4748,6 +4802,54 @@ class CommonDatautils { } + static def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy new file mode 100644 index 000000000..e46528404 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -0,0 +1,312 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PoiLocation { + + def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity() + def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity() + def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges() + def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity() + + static main(args) { + def task = new PoiLocation(); + task.run(); + } + def run(){ + def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"] + for(def city : citys){ + findPois(city) + } + } + + + def findPois(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("run_status", new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("_id", 1) + keys.put("province", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("township", 1) + keys.put("location", 1) + + def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def rets = [] + def index = 0 + for(def d : dbcursor){ + rets << d + if(rets.size()>=10000){ + index++ + def datas = checkData(index, rets) +// updateToEs(datas) + updateToEs2(datas) + updateToMG(datas) + rets = [] + print "." + } + } + return rets; + } + + def checkData(def index, def datas){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + + def updateToMG(rows){ + def rets = [] + for(def i=0;i + if(k!="id"){ + udata.put(k,v) + } + } + def q = new BasicDBObject() + q.put("_id", m.id) + MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy new file mode 100644 index 000000000..4d77b4cdf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels + +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class Test{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) +// 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + + static main(args) { + def task = new Test() + task.run() + } + + def run(){ + def i=1 + while(i<2){ + def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); + def rets = process(districts) + println i + " "+districts.size() +" "+rets.size() + i++ + } + + } + + def process(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+d.town+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + + + def updateQuery = new BasicDBObject("city", city).append("street",name) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + + def checkTown(){ + def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + for(def ad : dbcursor){ + def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street) + def count = TOWNPOLYLINE_COLL.count(polyline_q) + if(count==1){ + println ad.city+" "+ ad.street+" "+count + + def d = TOWNPOLYLINE_COLL.findOne(polyline_q) + if(d){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("polyline_source", "区域+街道匹配") + + def updateQuery = new BasicDBObject("_id", ad._id) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + } + } + + + def updateTownshipByRegion(){ + def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域") + def keys = new BasicDBObject() + keys.put("mzb_district_name",1) + keys.put("mzb_district_code",1) + + keys.put("city",1) + keys.put("district",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys) + for(def d : dbcursor){ + def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city) + def udata = new BasicDBObject() + udata.put("mzb_district_name",d.mzb_district_name) + udata.put("mzb_district_code",d.mzb_district_code) + ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true) + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy new file mode 100644 index 000000000..f6e830017 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -0,0 +1,324 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TownshipStrandUpdate { + + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + def fullCityMap = null + def regionMap = null + def commMap = null + def polylineMap = null + def newAdvMappingMap = null + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new TownshipStrandUpdate() + task.process() + + } + + + void process() { + fullCityMap = CommonDatautils.queryFullNameByCity() + regionMap = CommonDatautils.queryDistNameByCity() + commMap = CommonDatautils.queryCommunityByCity() + polylineMap = CommonDatautils.findAllTownRanges() + newAdvMappingMap = CommonDatautils.queryNewTownByCity() +// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + for(def city : citys){ + if(["中山"].contains(city)){ + continue + } + def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + if(!file.exists()){ + def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) + dists = checkData(dists) + def datas = createDatas(dists) + ExcelUtils.write(fileName, datas, "${city}") + } + } + } + + def checkData(def dists){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i maxScore){ + maxScore = score + town = townObj + } + } + if(town){ + town.score = maxScore + } + return town + } + + + def getSimilarTown(regions, fullCity, townstr){ + def town = null + for(def r : regions){ + def townList = commMap.get(fullCity.city+"#"+r) + + for(def townObj : townList){ + if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){ + town = townObj + }else if(townObj.alias && townObj.alias.contains(townstr)){ + town = townObj + }else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){ + def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","") + if(townstr.contains(newTown)){ + town = townObj + } + } + if(town){ + break + } + } + } + return town + } + + + def getFullRegion(fullCity, d){ + def regions = regionMap.get(fullCity.city) + def names = [] + if(regions){ + for(def r : regions){ + if(r.mzb_district_name.contains(d.region)){ + names << r.district + } + } + } + return names; + } + + + def createDatas(dists){ + def rets = [] + rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"] + for(def d : dists){ + rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score] + } + return rets; + } + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy index 7c9472e42..7d34463a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -1,13 +1,46 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection class TradeDatasDistBatch { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com") static main(args) { -// def + def task = new TradeDatasDistBatch() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d-> + + def udata = new BasicDBObject() + udata.put("city", d['A']) + udata.put("region", d['B']) + udata.put("name", d['C']) + udata.put("alias", d['D']) + alias_fang_COLL.save(udata) +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// def flag = null +// if(!dist){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district") +// if(rets && rets.size()==1){ +// flag = "别名匹配" +// dist = rets[0] +// } +// }else{ +// flag = "名称匹配" +// } +// if(flag){ +// d['F'] = dist.city +// d['G'] = dist.region +// d['H'] = dist.name +// } + }) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index 07d8e5679..63f75b256 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch { def stop = false def month = -1 while(!stop){ - if(sDate<202110){ + if(sDate<202004){ def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) println weekend.toString() +" "+sDate runDatas(citys) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index dbbf916d8..f0c4282e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,10 +33,10 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" -// addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" + addGXDPrice(filePath) // addCcbPrice(filePath) - addThirdPrice(filePath) +// addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() @@ -241,12 +241,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['L']) { - def dist = CommonDatautils.findDistrictById(d['E']) + if (!d['M']) { + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['L'] = gxdPrice + d['M'] = gxdPrice } } }) @@ -270,16 +270,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['N']){ - if(d['C'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['C']) + if(!d['L']){ + if(d['B'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ // d['W'] = dbo.avg_price // d['X'] = dbo.cal_type - d['N'] = dbo.avg_price_2b + d['L'] = dbo.avg_price_2b } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index 924db2def..35f821ce3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '上海市奉贤区星火开发区莲塘路251号8幢', + 'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 0682ee62c..63996e918 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -80,13 +80,13 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() - sysnchVillage() +// sysnchVillage() -// addTownship() + addTownship() } def addTownship(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d-> if(d['E']){ // def gaodeMap = getTownshipInfo(d['F']+","+d['E']) // if(gaodeMap && gaodeMap.township){ @@ -114,7 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{ d['L'] = "是" } } - }) + },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 19c0faebf..907e2a572 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -98,6 +98,23 @@ class InsidePolygon { } return list; } + + public static List> createPoint4(def pointList){ + List> alllist = new ArrayList<>(); + for(def pointArr : pointList){ + List list = new ArrayList<>(); + for(def arr : pointArr){ + Point point = new Point(); + double x = arr[0]; + double y = arr[1]; + point.setLon(x); + point.setLat(y); + list.add(point); + } + alllist << list + } + return alllist + } /** * 判定一个经纬度点与区域的关系 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index f14bedadd..6aba12207 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -76,7 +76,6 @@ class ElasticUtils { connection.close() } - static RestHighLevelClient getClient(String host) { RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { @@ -101,6 +100,30 @@ class ElasticUtils { } + static RestHighLevelClient getClient89(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + /** * 创建一个ES批处理器 * @param host diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 7e85c66ad..50882747b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 5 + def nThread = Runtime.getRuntime().availableProcessors() * 4 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index ab5c90233..b3b119384 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995 /** * 经纬度转地址
From efc6cfea74d99a15ec6e568d42299750b74397e6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 18 Jun 2024 19:32:21 +0800 Subject: [PATCH 28/29] print log --- .../com/fangjia/script/gj/data_handle/PoiLocation.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index e46528404..dbe071346 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -305,7 +305,12 @@ class PoiLocation { } def q = new BasicDBObject() q.put("_id", m.id) - MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) + if(udata){ + MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) + }else{ + println m+" 无法修改-无值" + } + } } From e77648cd2918e22816da2509be833939de7f208f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 18 Jun 2024 19:41:00 +0800 Subject: [PATCH 29/29] update print log --- .../groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index dbe071346..8edf46edf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -308,7 +308,7 @@ class PoiLocation { if(udata){ MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) }else{ - println m+" 无法修改-无值" + println m.toString()+" 无法修改-无值" } }