diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index dca91ddd9..f4594ad70 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -177,8 +177,8 @@ public class BaseAvgImportServlet extends HttpServlet { String realName) { int totalNum = sheet.getLastRowNum(); System.out.println("上传文件共:" + totalNum + "个"); - if (totalNum > 1000) { - totalNum = 1000; + if (totalNum > 5000) { + totalNum = 5000; } Integer totalLine = 0;// 总行数 diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java index 00f40baa6..bcb042e67 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java @@ -257,4 +257,9 @@ public class GeoUtils { ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0; return ret; } + + public static void main(String[] args){ + Map location = gps84ToGcj02(31.218024629574998,121.36562884455); + System.out.println(location.toString()); + } } diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java index b49974a26..ccc5285c3 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java @@ -98,5 +98,13 @@ public class SolutionController { public Object standardizationAction(FangjiaModel model) { return model; } - + + + /** + * 法拍房解决方案 + * @param model + * @return + */ + public Object paimaiAction(FangjiaModel model){return model;} + } diff --git a/projects/fangjia-coresite/src/main/webapp/app/index.ftl b/projects/fangjia-coresite/src/main/webapp/app/index.ftl index 3d46edff9..bd01a269a 100644 --- a/projects/fangjia-coresite/src/main/webapp/app/index.ftl +++ b/projects/fangjia-coresite/src/main/webapp/app/index.ftl @@ -40,6 +40,8 @@ .footer{height:110px;background:#515151;} .footer > p{font-size:12px;color:#D7D7D7;text-align:center;line-height:18px;padding-top:38px;} .footer > p label{color:#D7D7D7;} + + + + + + + + + + + + + +<#include "/header/v5/_header.ftl"/> + + + + + +
+

鏈嶅姟浠嬬粛

+
+
+ +
+
+ +
+
+
+

鍙告硶鎷嶅崠甯傚満鏄笉鑹祫浜у缃殑缁堢锛屽叾浜ゆ槗閲戦銆佹暟閲忎环鏍肩瓑淇℃伅鍏锋湁閲嶈浠峰�笺�傞殢鐫�浜掕仈缃戠殑鏅強锛岀綉缁滃徃娉曟媿鍗栨棩鐩婃垚涓哄徃娉曟媿鍗栫殑涓绘祦褰㈠紡銆傞�氳繃瀵瑰悇涓昏缃戠粶鎷嶅崠骞冲彴鐨勫叕寮�鏁版嵁杩涜缁熻鍒嗘瀽锛屽彲浠ュ弽鏄犲嚭鍚勫湴鐨勬硶鎷嶆埧甯傚満瑙勬ā銆佹爣鐨勪环鏍笺�佷氦鏄撶儹搴︽儏鍐碉紝棰勫垽鍚勭被鎴夸骇鍦ㄦ媿鍗栧競鍦虹殑娴佸姩鎬у拰鎴愪氦鎶樻墸鐜囧彉鍖栬秼鍔匡紝浣滀负閾惰鎶垫娂鎴夸骇浠峰�艰瘎浼板拰璐峰悗椋庢帶鐨勯噸瑕佷緷鎹��

+
+
+
+
+

鍙告硶鎷嶅崠甯傚満鏄笉鑹祫浜у缃殑缁堢锛屽叾浜ゆ槗閲戦銆佹暟閲忎环鏍肩瓑淇℃伅鍏锋湁閲嶈浠峰�笺�傞殢鐫�浜掕仈缃戠殑鏅強锛岀綉缁滃徃娉曟媿鍗栨棩鐩婃垚涓哄徃娉曟媿鍗栫殑涓绘祦褰㈠紡銆傞�氳繃瀵瑰悇涓昏缃戠粶鎷嶅崠骞冲彴鐨勫叕寮�鏁版嵁杩涜缁熻鍒嗘瀽锛屽彲浠ュ弽鏄犲嚭鍚勫湴鐨勬硶鎷嶆埧甯傚満瑙勬ā銆佹爣鐨勪环鏍笺�佷氦鏄撶儹搴︽儏鍐碉紝棰勫垽鍚勭被鎴夸骇鍦ㄦ媿鍗栧競鍦虹殑娴佸姩鎬у拰鎴愪氦鎶樻墸鐜囧彉鍖栬秼鍔匡紝浣滀负閾惰鎶垫娂鎴夸骇浠峰�艰瘎浼板拰璐峰悗椋庢帶鐨勯噸瑕佷緷鎹��

+
+
+ +
+
+ + + + + +
+

娉曟媿鎴挎暟鎹垎鏋愭渚�

+
+ + +
+
+
+

+

2022骞翠笂鍗婂勾锛孹X甯傛硶鎷嶆埧甯傚満鍏辫鎸傛媿涓婃媿鎴挎簮鏈�1265濂楋紝鎴愪氦娉曟媿鎴�482濂楋紝鎴愪氦鐜囦负38%锛堥櫎鍘绘挙鎷嶅鏁帮級銆傚钩鍧囨垚浜ゆ姌鎵f姌73%锛屽叾涓綇瀹呮垚浜ゆ姌鎵g巼88%锛屽埆澧呮姌鎵g巼涓�65%锛屽晢鍔炴姌鎵g巼涓�60%锛屽叡璁¢噴鏀�15.28浜垮厓璐埧宸环銆�

+ +

+

XX鍖烘硶鎷嶆埧甯傚満2022骞�1-6鏈圶X鍖哄徃娉曟媿鍗栧競鍦烘寕鎷嶄綇瀹呯敤鎴挎埧婧�92濂楋紝鎴愪氦48濂楋紝鎴愪氦鐜�52.2%锛屾垚浜ゅ潎浠�22450鍏冿紝浣忓畢鎴愪氦鎶樻墸7.8鎶橈紱鎸傛媿鍟嗕笟鐢ㄦ埧鎴挎簮364濂楋紝鎴愪氦113濂楋紝鎴愪氦鐜�31%锛屾垚浜ゅ潎浠�44168锛屾垚浜ゆ姌鎵�9.4鎶橈紱宸ヤ笟鐢ㄦ埧鍜屽湡鍦版棤鎸傛媿銆�

+
+ +
+

+

2022骞翠笂鍗婂勾锛孹X甯傛硶鎷嶆埧甯傚満鍏辫鎸傛媿涓婃媿鎴挎簮鏈�1265濂楋紝鎴愪氦娉曟媿鎴�482濂楋紝鎴愪氦鐜囦负38%锛堥櫎鍘绘挙鎷嶅鏁帮級銆傚钩鍧囨垚浜ゆ姌鎵f姌73%锛屽叾涓綇瀹呮垚浜ゆ姌鎵g巼88%锛屽埆澧呮姌鎵g巼涓�65%锛屽晢鍔炴姌鎵g巼涓�60%锛屽叡璁¢噴鏀�15.28浜垮厓璐埧宸环銆�

+ +

+

XX鍖烘硶鎷嶆埧甯傚満2022骞�1-6鏈圶X鍖哄徃娉曟媿鍗栧競鍦烘寕鎷嶄綇瀹呯敤鎴挎埧婧�92濂楋紝鎴愪氦48濂楋紝鎴愪氦鐜�52.2%锛屾垚浜ゅ潎浠�22450鍏冿紝浣忓畢鎴愪氦鎶樻墸7.8鎶橈紱鎸傛媿鍟嗕笟鐢ㄦ埧鎴挎簮364濂楋紝鎴愪氦113濂楋紝鎴愪氦鐜�31%锛屾垚浜ゅ潎浠�44168锛屾垚浜ゆ姌鎵�9.4鎶橈紱宸ヤ笟鐢ㄦ埧鍜屽湡鍦版棤鎸傛媿銆�

+
+
+
+ + +
+
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+
+ + +
+

+

濡傛兂鑾峰彇鎮ㄦ墍鍏虫敞鍩庡競鐨勬硶鎷嶆埧甯傚満鍒嗘瀽鎶ュ憡锛屾垨鑰呮煡璇换涓�妤肩洏鐨勬硶鎷嶆垚浜ゆ姌鎵g巼锛岃鑷寸數  400-188-3787  鑱旂郴鎴戜滑锛屾垨鑰呯暀涓嬫偍鐨勮仈绯绘柟寮忓拰闇�姹傦紝鎴戜滑浼氱涓�鏃堕棿鑱旂郴鎮ㄣ��

+ 绔嬪嵆浣撻獙 +
+
+

+

濡傛兂鑾峰彇鎮ㄦ墍鍏虫敞鍩庡競鐨勬硶鎷嶆埧甯傚満鍒嗘瀽鎶ュ憡锛屾垨鑰呮煡璇换涓�妤肩洏鐨勬硶鎷嶆垚浜ゆ姌鎵g巼锛岃鑷寸數鑱旂郴鎴戜滑锛屾垨鑰呯暀涓嬫偍鐨勮仈绯绘柟寮忓拰闇�姹傦紝鎴戜滑浼氱涓�鏃堕棿鑱旂郴鎮ㄣ��

+ 绔嬪嵆浣撻獙 +
+ +
+ +
+
+
+

+

娆㈣繋浜茶嚜浣撻獙CPDB鎴夸骇澶ф暟鎹簱锛屾偍鍙�氳繃鍦ㄧ嚎鏂瑰紡鐢宠娴嬭瘯鏁版嵁鎴栬仈绯绘暟鎹【闂紝浠栦滑灏嗛潪甯镐箰鎰忎负鎮ㄧ殑鏁版嵁闇�姹傞噺韬畾鍒跺悎閫傜殑瑙e喅鏂规銆�

+
+ +
+
+ +<#include "/header/v5/_footer.ftl"/> + + + + + diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java index b389ce299..4eb63c496 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java @@ -27,8 +27,8 @@ public enum NewResourceRecommendEnum { API_GUZ_HI("1019", "AMP估价管理平台"), FJT("1020", "房价通"), FANG_JIA_ZOU_SHI("1021", "房价走势"), - FDANG_DIAO_BAO_GAO("1022", "房调报告"); - + FDANG_DIAO_BAO_GAO("1022", "房调报告"), + FA_PAI_FANG("1023", "法拍房数据解决方案"); NewResourceRecommendEnum(String id, String value) { diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java index 6cb65c498..54093da2f 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java @@ -23,7 +23,8 @@ public enum SalesLeadEnum { FDANG_DIAO_BAO_GAO("1017", "房产专业调查评估报告"), BUSINESS_ASK("1018", "业务咨询"), HOUSE_EVAL("1019", "房产评估"), - DIDATA("1020", "地数平台"); + DIDATA("1020", "地数平台"), + FA_PAI_FANG("1023", "法拍房数据解决方案"); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index b0fc86c99..adb32083b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -649,7 +649,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i jsonList.add(temp); } - if(StringUtils.isNotBlank(query.getSort())){ + if(StringUtils.isNotBlank(query.getSortField())){ JSONObject jsonOrder = new JSONObject(); if(query.getOrder()){ jsonOrder.put("order", "asc"); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java index 40a303f11..d5d69a1af 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java @@ -386,24 +386,24 @@ public class FshController { Double oneMonthMarkup = null; Evaluation evaluation = null; if(model.isYouDa()){ - if("上海".equals(model.getCity())){ - YdDealPrice ydDealPrice = ydDealPriceService.findYdDealPrice(model.getCity(), model.getDistrict(), model.getName()); - if(ydDealPrice == null){ - mav.setModel(BasicModel.buildOKModel().setMsg("该小区暂不支持估价").markNoData()); - return mav; - }else{ - evaluation = evaluationService.ydEvaluate(district, buildYdEvaluation(model, request), model.getCalDate(),ydDealPrice.getPrice()); - //近30天的涨跌幅:取最近1个月至3个月之间最近的那期价格和当前均价相比 - Date end = DateUtil.getDateOfLastMonth(new Date(),-1); - Date start = DateUtil.getDateOfLastMonth(new Date(),-3); - List ydDealPrices = ydDealPriceService.findYdDealPrices(model.getCity(), model.getDistrict(), model.getName(),start,end); - if(ydDealPrices!=null && ydDealPrices.size()>0){ - oneMonthMarkup = (double)(ydDealPrices.get(0).getPrice()-ydDealPrice.getPrice())/ydDealPrice.getPrice(); - } - } - }else{ +// if("上海".equals(model.getCity())){ +// YdDealPrice ydDealPrice = ydDealPriceService.findYdDealPrice(model.getCity(), model.getDistrict(), model.getName()); +// if(ydDealPrice == null){ +// mav.setModel(BasicModel.buildOKModel().setMsg("该小区暂不支持估价").markNoData()); +// return mav; +// }else{ +// evaluation = evaluationService.ydEvaluate(district, buildYdEvaluation(model, request), model.getCalDate(),ydDealPrice.getPrice()); +// //近30天的涨跌幅:取最近1个月至3个月之间最近的那期价格和当前均价相比 +// Date end = DateUtil.getDateOfLastMonth(new Date(),-1); +// Date start = DateUtil.getDateOfLastMonth(new Date(),-3); +// List ydDealPrices = ydDealPriceService.findYdDealPrices(model.getCity(), model.getDistrict(), model.getName(),start,end); +// if(ydDealPrices!=null && ydDealPrices.size()>0){ +// oneMonthMarkup = (double)(ydDealPrices.get(0).getPrice()-ydDealPrice.getPrice())/ydDealPrice.getPrice(); +// } +// } +// }else{ evaluation = evaluationService.evaluate(district, buildYdEvaluation(model, request), model.getCalDate()); - } +// } }else{ evaluation = evaluationService.evaluate(district, buildEvaluation(model, request), model.getCalDate()); } @@ -1163,18 +1163,18 @@ public class FshController { private Map buildDistrictAvgZouShiByMonth(DistrictModel model,DistrictKey districtKey, Date end, Date start) { Map> avgPriceDetailMap = new HashMap>(); - if(model.isYouDa() && "上海".equals(model.getCityName())){ - List list = ydDealPriceService.findYdDealPrices(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end); - for (YdDealPrice stat : list) { - String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM"); - List prices = avgPriceDetailMap.get(ws); - if(null == prices){ - prices = new ArrayList(); - avgPriceDetailMap.put(ws, prices); - } - prices.add(stat.getPrice()); - } - }else{ +// if(model.isYouDa() && "上海".equals(model.getCityName())){ +// List list = ydDealPriceService.findYdDealPrices(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end); +// for (YdDealPrice stat : list) { +// String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM"); +// List prices = avgPriceDetailMap.get(ws); +// if(null == prices){ +// prices = new ArrayList(); +// avgPriceDetailMap.put(ws, prices); +// } +// prices.add(stat.getPrice()); +// } +// }else{ List list = districtStatService.findDistrictPrice(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end); for (DistrictStat stat : list) { String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM"); @@ -1185,7 +1185,7 @@ public class FshController { } prices.add(stat.getAvgPrice()); } - } +// } Map districtMap = new HashMap(); if(null != avgPriceDetailMap){ diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 82bf9b1ce..078ca3b0f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1235,7 +1235,7 @@ public class WeiXinEvaluationController { Point point = new Point(Double.valueOf(ll[0]), Double.valueOf(ll[1])); points.add(point); } - Point point = new Point(district.getLng()/10000000000.0, district.getLat()/10000000000.0); + Point point = new Point(district.getLng2()/10000000000.0, district.getLat2()/10000000000.0); //boolean flag = InsidePolygon.aboutZone(point, points); //boolean flag = InsidePolygon.isPtInPoly(point.getLon(), point.getLat(), points); boolean flag = InsidePolygon2.checkIsInPolygon(point, points); diff --git a/projects/fangjia-open/src/main/webapp/static/js/report.js b/projects/fangjia-open/src/main/webapp/static/js/report.js index b61a6e2d1..f3d833ccf 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/report.js +++ b/projects/fangjia-open/src/main/webapp/static/js/report.js @@ -362,7 +362,7 @@ $('.formal_btn').on('click', function () { }); function clickFormalReport(){ - $.prompt('姝e紡鎶ュ憡灏嗗敖蹇偖瀵勭粰鎮紝璇锋敞鎰忔煡鏀讹紝璋㈣阿锛佸鏈夌枒闂彲涓庢垜浠仈绯伙細62392317', '璇峰~鍐欐寮忔姤鍛婇偖瀵勫湴鍧�', function (value) { + $.prompt('姝e紡鎶ュ憡灏嗗敖蹇偖瀵勭粰鎮紝璇锋敞鎰忔煡鏀讹紝璋㈣阿锛� ', '璇峰~鍐欐寮忔姤鍛婇偖瀵勫湴鍧�', function (value) { var address = $(".modal-text-input").val(); if(address==""){ $.alert('鍦板潃涓嶈兘涓虹┖'); diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index f16637105..eaab26cb0 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -37,14 +37,13 @@

${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("鍙�")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}鍙�${fjtEvaluation.roomNum!}瀹� 閲嶆柊璇勪及 >

*璇风‘璁や笌浜ц瘉鍦板潃涓�鑷�

- <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0> -

璇勪及鎬讳环锛�<#if fjtEvaluation??> - ${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

-

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.inquirePrice gt 0>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

+ <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0> +

璇勪及鎬讳环锛�<#if fjtEvaluation??><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

+

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

<#if role?? && role==17 && loanInfoMap??>

鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

鍙捶鎴愭暟锛�${loanInfoMap.loan_ratio!}鎴�

-

鍙捶閲戦锛�${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

+

鍙捶閲戦锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

璐锋鍒╃巼锛�4.5%

<#else> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 6ef05438e..3b92e612c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_all; def rows = [] def headers = [] for (def city : citys) { @@ -69,7 +69,7 @@ class BigdataBuildingBatch extends BaseBatch{ q.put("city", city) q.put("status", 1) // q.put("name","辽源一村") - def dbcursor = LOU_DONG_STAT.find(q) + def dbcursor = LOU_DONG_STAT.find(q).batchSize(5000) def map = [:] def set = new HashSet() for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 1000b157c..066318e29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -28,7 +28,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index b54b97e70..c06cef98e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -35,7 +35,8 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_92; + def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -43,18 +44,19 @@ class BigdataCommunityBatch extends BaseBatch{ //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + println "starting 。。。。。。。。。" for(def city : citys){ + index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) - println city +" 小区量:"+dists.size() def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) - println city +" "+rets.size() - + println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } @@ -220,7 +222,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "" ret << 0 }catch(Exception e){ - println dist.city + " "+dist.name + println dist.city + " "+dist.name+" error" e.printStackTrace() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 62c7e0981..3456607da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap @@ -32,8 +33,11 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ +// def startDate = 202206 +// def endDate = 202208 + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{ DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -52,49 +57,52 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) - def citys = ['南京', '无锡', '徐州', '常州', '苏州', '南通', '连云港', '淮安', '盐城', '扬州', '镇江', '泰州', '宿迁']//CcbCommonUtils.citys; -// def ccbCitys = CcbCommonUtils.citys; - - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } +// def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) + if(args && args.size()>=2){ + curDate = args[1].toString().toInteger() } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + // for(def city : citys){ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// -// def trades = getCityData(city, statMap, ppeDists) -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成" + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + def trades = getCityData(city, statMap, ppeDists) + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + println "txt 生成 " + rows.size() } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -102,6 +110,9 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) +// def lianjiaPriceMap = findThirdPartyPrice("上海") + def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') + def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) for(def i=0;i 3000) { + price = db.lianjia_esf_price + } + if (!price && db.beike_esf_price && db.beike_esf_price > 3000) { + price = db.beike_esf_price + } + if(price){ + map.put(db.region+"##"+db.name, price) + } + + } + return map + } + + def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){ def ret = [] def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; } - ret << cityCodeDatas.get(trade.city) - ret << trade._id.toString() - ret << trade.city - def data = distCodeMap.get(trade.fjRegion) - if(data){ - ret << data.district_code - ret << data.district - }else{ - ret << "" - ret << "" - } - def street = "" - def streetNumber = "" - if(trade.address){ - if(trade.address.toString().contains("弄")){ - String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) - if(number){ - street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) - streetNumber = number + def price = lianjiaPriceMap.get(obj.region+"##"+obj.name) + if(price){ + def mod = java.lang.Math.abs((trade.price-price)*100/price) + if(mod<=10){ + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << trade.city + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" } - } - if(!street){ - def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) - if(jsonObj && jsonObj.searchAddress){ - if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + def street = "" + def streetNumber = "" + if(trade.address){ + if(trade.address.toString().contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) streetNumber = number } - }else{ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber } + if(!street){ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } + } } - } - } - if(!street){ - if(obj.address){ - if(obj.address.toString().contains("弄")){ - String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) - if(number){ - street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) - streetNumber = number + if(!street){ + if(obj.address){ + if(obj.address.toString().contains("弄")){ + String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) + if(number){ + street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) + if(jsonObj && jsonObj.searchAddress){ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } } + } + + ret << street + ret << streetNumber + + + if(obj){ + ret << obj._id.toString() + ret << trade.fjName + } + + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) + ret << ""//期数 + String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) + if(roomNum && roomNum.contains("0")){ + ret << roomNum.substring(0, roomNum.lastIndexOf("0")) }else{ - def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) - if(jsonObj && jsonObj.searchAddress){ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber - } + ret << trade.floor } + + ret << "" + ret << trade.height + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.price + ret << trade.totalPrice * 10000 + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "99" + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.totalPrice * 10000 + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") } } - ret << street - ret << streetNumber - - - if(obj){ - ret << obj._id.toString() - ret << trade.fjName - } - - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 - ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) - ret << ""//期数 - String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) - if(roomNum && roomNum.contains("0")){ - ret << roomNum.substring(0, roomNum.lastIndexOf("0")) - }else{ - ret << trade.floor - } - - ret << "" - ret << trade.height - def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 - def buildType = '' - if(arrs){ - buildType = arrs[0] - } - ret << buildType - ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) - ret << trade.area - ret << "" - ret << getHouseRoom(buildType, trade.room, trade.hall) - ret << trade.room - ret << trade.hall - ret << "" - ret << trade.toilet - ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) - ret << "" - ret << CcbCodeUtils.getDirectionCode(trade.direction) - ret << CcbCodeUtils.getFitmentCode(trade.fitment) - ret << "" - ret << trade.price - ret << trade.totalPrice * 10000 - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") - ret << "02" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "99" - - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << trade.totalPrice * 10000 - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") return ret; } @@ -289,7 +330,7 @@ class BigdataDealBatch extends BaseBatch{ } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() - ret << trade.city + ret << CcbCommonUtils.getCcbCity(trade.city) if(data){ ret << data.district_code ret << data.district @@ -543,11 +584,14 @@ class BigdataDealBatch extends BaseBatch{ def getDistrictStatByCity(city, weekend){ def q = new BasicDBObject() q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) + q.put("weekend", new BasicDBObject('$gte', weekend)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1)) def map = new ConcurrentHashMap() for(def d : dbcursor){ - map.put(d.region+"@"+d.district_name, d.avg_price) + def price = map.get(d.region+"##"+d.district_name) + if(!price && d.value_type==2){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } } return map } @@ -572,7 +616,7 @@ class BigdataDealBatch extends BaseBatch{ // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) final CountDownLatch latch = new CountDownLatch(trades.size()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index c3affa2bd..e5272b8bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql + /** * 建行模型估值项目 * 区域数据同步 @@ -17,33 +19,74 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class BigdataDistrictBatch extends BaseBatch{ + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { runBatch(new BigdataDistrictBatch(), args); } @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; - def rows = [] - for(def city : citys){ - def datas = findDistrictCodeValue(city) - if(datas){ - rows.addAll(datas) - } - println city+" "+ datas.size() - } - - def allDatas = createDatas(rows) - - def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] - + def datas = readText("D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.txt") def rets = [] - rets << headers - rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); + def index = 0 + for(def line : datas){ + def arrs = line[0].split("\\|@\\|") + def sql = "select count(*) as count from BIGDATA_DISTRICT_GJ where distid='${arrs[0]}'"; + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + if(count==0){ + rets << line + } + } + writeTxt(rets, "D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ_add.txt") +// def citys = CcbCommonUtils.citys_all; +// def rows = [] +// for(def city : citys){ +// def datas = findDistrictCodeValue(city) +// if(datas){ +// rows.addAll(datas) +// } +// println city+" "+ datas.size() +// } +// +// def allDatas = createDatas(rows) +// +// def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] +// +// def rets = [] +// rets << headers +// rets.addAll(allDatas) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") +// FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } + + def writeTxt(def rows, def path){ + StringBuilder builder = new StringBuilder() + for(def row : rows){ + builder.append(row) + builder.append('\n') + } + org.apache.commons.io.FileUtils.writeStringToFile(new File(path), builder.toString(), "GBK") + } + private def readText(def fileName){ + File file = new File(fileName) + def ret = []; + //对文件内容的每一行进行分割处理,比较常用在处理csv文件 + file.splitEachLine(" ") { + def temp = it + ret << temp; + } + //转为Writable对象,可重定向输出 + file.asWritable() + return ret; + } + + def createDatas(rows){ def allRows = [] def index = 1 @@ -72,16 +115,16 @@ class BigdataDistrictBatch extends BaseBatch{ def findDistrictCodeValue(city){ def q = new BasicDBObject() - q.put("fangjia_city", city) - // q.put("district", new BasicDBObject('$ne',"市辖区")) - def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + q.put("fjw_city", city) +// def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + def dbcursor = CcbCommonUtils.FANGJIA_AREA_CODE.find(q) def lists = [] for(def db : dbcursor){ if(db.district_code){ def m = [:] m.id = db._id.toString() m.dist_name = db.district - m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") + m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fjw_district, "") m.city_id = db.city_code m.dist_code = db.district_code lists << m diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 12a02fabb..ca517f31a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -67,20 +67,39 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { def index = 0 - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) + statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } } + + def statCity(def citys){ + def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1) + def houseCitys = [] + for(def d : datas){ + houseCitys << d['C'] + } + for(def city : citys){ + if(!houseCitys.contains(city)){ + def scity = CcbCommonUtils.specialCityMap.get(city) + if(!scity){ + println city + } + } + } + } + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() @@ -538,7 +557,7 @@ class BigdataForsaleBatch extends BaseBatch{ } rowMap.houseId = seawater.id - rowMap.city = city + rowMap.city = CcbCommonUtils.getCcbCity(city) //rowMap.region =seawater.region //rowMap.address = seawater.address diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index 39a26972b..08653a46a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{ @Override public void process(Object args) { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def citys = CcbCommonUtils.citys; -// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" - def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) - def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) -// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) - def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) - def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) -// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) + def citys = CcbCommonUtils.citys_252 + + +// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1) +// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1) +// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1) +// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1) + + def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1) def rets = [] // def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // rets.addAll(rows1) - - def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") - rets.addAll(rows1) - - def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") - rets.addAll(rows2) // -// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") +// rets.addAll(rows2) +// +// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") // rets.addAll(rows3) - - def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") - rets.addAll(rows4) - - def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") - rets.addAll(rows5) +//// +// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") +// rets.addAll(rows4) // -// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") -// rets.addAll(rows6) + def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") + rets.addAll(rows6) - def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] + +// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") +// rets.addAll(rows5) +// + + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat" if(CommonDatautils.isWindow()){ @@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{ }else if(type =="丧葬服务"){ ret << "ob101" }else if(type =="学校"){ - if(d['F'].contains("中学")){ + if(d['B'].contains("中学")){ ret << "ed138" - }else if(d['F'].contains("小学")){ + }else if(d['B'].contains("小学")){ ret << "ed114" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index db3beb6ef..46aa81fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,12 +16,16 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] -// public static def citys = ["武汉"] + public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] + public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] - public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") +// public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") + + public static final DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") + public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") @@ -34,39 +38,76 @@ class CcbCommonUtils { public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"] + public static DSCollection dataStoreConnection = null; public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def getRunningCitys(def args){ + def citys = [] + println "args 参数:"+args.toString() + if(args){ + def countStr = args[0] + if(countStr=="344"){ + citys = CcbCommonUtils.citys_all + }else if(countStr=="92"){ + citys = CcbCommonUtils.citys_92 + }else if(countStr=="252"){ + citys = CcbCommonUtils.citys_252 + } + }else{ + citys = CcbCommonUtils.citys_92 + } + println "运行城市数量:"+citys.size() + return citys; + } + + public static def findCityCodeValue(){ - def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] + def cityCodeMap = [:] + FANGJIA_AREA_CODE.find().each {it-> + def code = cityCodeMap.get(it.fjw_city) + if(!code){ + cityCodeMap.put(it.fjw_city, it.city_code.toString()) + } + } +// def oldCityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] return cityCodeMap; } + public static def getCcbCity(def city){ + if(city && specialCityMap.containsKey(city)){ + return specialCityMap.get(city) + } + return city; + } + public static def findAllDistrictCodeValue(city){ def q = new BasicDBObject() - q.put("fangjia_city", city) + q.put("fjw_city", city) //q.put("district", new BasicDBObject('$ne',"市辖区")) - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - map.put(db.fangjia_district, db) + map.put(db.fjw_district, db) } return map; } public static def findAllCityDistrictCodeValue(city){ def q = new BasicDBObject() - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - def distMap = map.get(db.fangjia_city) + def distMap = map.get(db.fjw_city) if(!distMap){ distMap = [:] - map.put(db.fangjia_city, distMap) + map.put(db.fjw_city, distMap) } - distMap.put(db.fangjia_district, db) + distMap.put(db.fjw_district, db) } return map; } @@ -77,8 +118,16 @@ class CcbCommonUtils { q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); def dbcursor = GUIDING_PRICE_COLL.find(q) def rets = [] + def sets = new HashSet() for(def d : dbcursor){ - rets << d + def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd') + def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date + if(!sets.contains(key)){ + rets << d + sets << key + }else{ + println key + } } return rets } @@ -368,7 +417,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","辽源一村") +// q.put("name","云山尚城") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 74474aff6..6b972553e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpUtils { - static def ftpServer = "192.168.0.45" + static def ftpServer = "192.168.0.45"//192.168.0.45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 13d0f6c73..a96478424 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -370,24 +370,24 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def publishTime = '2021-10-01' - def month = 202206 + def citys = CcbCommonUtils.citys_92 + def month = 202209 def rows = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202207) + def rets = findPeGteThirdParty(city, 202210) def total = 0 println city +" "+rets.size() for(def m : rets){ - def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" + def cityId = cityCodeDatas.get(city) + def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } } } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25" def rets = [] sqlInstance236.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 94741174c..ed5ed5f03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -62,7 +62,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ ret << dist._id.toString() ret << dist.name ret << cityCodeDatas.get(dist.city) - ret << dist.city + ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district ret << ps.avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 08ab70333..fed4c92bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -15,31 +15,39 @@ class ExportOracleData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null + static main(args) { def task = new ExportOracleData() - task.run() + task.run(args) } - def run() { + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) // def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() + exportBigdataPoiData()// + exportBigdataDistrictData()// - exportGuidedPrice()//随人工改变 - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // //处理小区情况 - exportBigdataCommunity() - exportCommunityStaging() - exportCommunityBuileType() + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// - exportThirdPartyPrice2(lastMonth, new_dists) - exportBigdataForsale("2022-06-19", new_dists) - exportBigdataDeal("2022-06-01", new_dists) + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); //计算结果相关 @@ -93,26 +101,61 @@ class ExportOracleData { } - def exportBigdataPoiData() { - def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj"; + def exportWebsiteRate(){ + def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] - data << r.CITYID - data << r.DISTID - data << r.ID - data << r.POI_NAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.DATA_SOURCE_MAP + data << r.WEIGHT data << r.TYPE - data << r.TYPE_CODE - data << r.LATITUDE - data << r.LONGITUDE - data << r.REMARK + data << r.SOURCECODE + data << r.CITYID + data << r.CITYNAME data << r.CREATETIME data << r.UPDATETIME - data << r.SOURCECODE allRows << data }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" @@ -128,8 +171,9 @@ class ExportOracleData { } def exportCommunityStaging() { - def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + def allRows = [] + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -158,9 +202,9 @@ class ExportOracleData { // def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def allRows = [] -// for(def city : citys){ -// def cityid = cityCodeDataMap.get(city) - def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.CITYID @@ -173,7 +217,8 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); -// } + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" @@ -189,28 +234,31 @@ class ExportOracleData { } def exportBigdataDistrictData() { - - def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.LJXZQH - data << r.LJXZQHDM - data << r.XZQHDM - data << r.WORKDATE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -227,26 +275,30 @@ class ExportOracleData { def exportGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.DEALDATE - data << r.AVG_PRICE - data << r.GUIDING_PERSON - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -261,25 +313,29 @@ class ExportOracleData { } def exportCommunityGuidedPrice(def date) { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.AVGPRICE - data << r.SOURCETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -296,30 +352,33 @@ class ExportOracleData { def exportBigdataDeal(def dealDate, def new_dists) { def allRows = [] - if(new_dists){ - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" - def rets1 = dealData(sql1) + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) - def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where distid in ${newstr}" - def rets2 = dealData(sql2) - allRows.addAll(rets1) - allRows.addAll(rets2) - }else{ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows = dealData(sql) + } } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; @@ -328,8 +387,6 @@ class ExportOracleData { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } def headers = [] -//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DEAL_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) @@ -462,25 +519,6 @@ class ExportOracleData { def exportThirdPartyPrice2(price_date, new_dists) { - def allRows = [] - if(new_dists) { - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + - "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + - "price_date=${price_date} and distid not in ${newstr} "; - def rets1 = getThirdPartyDatas(sql1) - allRows.addAll(rets1) - - def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + - "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + - "distid in ${newstr} "; - def rets2 = getThirdPartyDatas(sql2) - allRows.addAll(rets2) - }else{ - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; - allRows = getThirdPartyDatas(sql) - } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -488,9 +526,37 @@ class ExportOracleData { // fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] -//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) } @@ -517,53 +583,58 @@ class ExportOracleData { } def exportBigdataCommunity(){ - def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + - "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.DISTID - data << r.COMMID - data << r.DISTNAME - data << r.COMMNAME - data << r.ADDRESS - data << r.LATITUDE - data << r.LONGITUDE - data << r.LANDAREA - data << r.BUILDAREA - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << r.BUILDYEAR - data << r.HOUSEUSE - data << r.ZTS - data << r.WORKDATE - data << r.UPDATEFLAG - data << r.UPDATETIME - data << r.COMMTYPE - data << r.COMMALIAS - data << r.STREET - data << r.STREETNO - data << r.IFNEW - data << "" - data << r.REMARK - data << r.CREATETIME - data << r.SOURCECODE - data << r.XZJDB - data << r.LJXZQHDM - data << r.LJXZQH - data << r.STATUS - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", - //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", - //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) @@ -572,18 +643,15 @@ class ExportOracleData { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 for(def city : citys){ - def cityCode = cityCodeDataMap.get(city) - + def cityCode = cityCodeMap.get(city) def allRows = [] if(new_dists){ @@ -601,7 +669,13 @@ class ExportOracleData { allRows = getHouseDatas(sql) } if(allRows.size()>0){ - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } } @@ -743,8 +817,8 @@ class ExportOracleData { fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", - //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", - //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "AVM_EVALUATION_GJNB 数据文件输出完成......" @@ -958,55 +1032,55 @@ class ExportOracleData { // // for(def city : citys) { // def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; - def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMUNITY_ID - data << r.CITYID - data << r.CITY_NAME - data << r.DISTRICT_CODE - data << r.DISTRICT_NAME - data << r.COMMUNITY_NAME - data << r.PQ_NAME - data << r.BASE_PRICE - data << r.CONFIDENCE_LEVEL - data << r.CNT - data << r.MIN_PRICE - data << r.MEDIANPRICE - data << r.MEANPRICE - data << r.MAX_PRICE - data << r.TIME - data << r.FIRSTDATAT - data << r.THIRDDATA - data << r.RELATIVEERROR - data << r.SKEWNESS - data << r.PQ_PRICE_RANGE - data << r.DISTRICT_PRICE_RANGE - data << r.DESCRIPTION - data << r.PRIV_BASE_PRICE - data << r.IS_ARTTFICIAL - data << r.PRIV_FINAL_PRICE - data << r.CREATE_DATE - data << r.PRICE - data << r.PRICE_SOURCE - data << r.CASE_MD_PRICE - data << r.CASE_AVG_PRICE - data << r.INDEX_LABEL - data << r.MAIN_PROPERTY - data << r.COMMTYPE - data << r.NEW_DEAL_PRICE - data << r.NEW_DEAL_DATE - data << r.PRICE_MOM_RATIO - data << r.MAX_MOM_RATIO - data << r.PRICE_YOY_RATIO - data << r.SATUR_RATIO - data << r.NEIGHBOR_PRICE - allRows << data - }); - println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); // } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy new file mode 100644 index 000000000..b28516075 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -0,0 +1,267 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql +import org.apache.commons.lang.StringUtils + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class FjwPricePkCcbPrice { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new FjwPricePkCcbPrice() +// task.run() +// task.updatePrice() +// task.addFjwPrice() +// task.addCcbPrice() + task.addMarketPrice() + } + + def addMarketPrice(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\拍卖南京.xlsx",{d-> + if(d['D']) { + d['K'] = d['D'] + }else if(d['H']){ + d['K'] = d['H'] + } + + }) + } + + def addCcbPrice(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d-> + if(d['B']){ + def cityId = cityCodeDataMap.get(d['B']) + def dist = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district") + if(dist){ + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209) + if(map){ + d['I'] = map.avg_price + d['J'] = map.calc_type + d['k'] = map.credit + } + } + +// def dist = CommonDatautils.findDistrictById(d['B']) +// if(dist){ +// d['J'] = dist.status +// } + } + }) + } + + + def updatePrice(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d-> + index++ + println index + def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd')) + if(dbo){ + d['G'] = dbo.future_value + } + }) + } + + def addFjwPrice(){ + ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d-> + if(!d['M']){ + def dist = CommonDatautils.findDistrictById(d['D']) + if(dist){ + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + if(dbo){ + d['M'] = dbo.avg_price + } + } + } + + }) + } + + def run(){ + def ccbPrices = getCommunityBasePriceGj(202208) + println "ccb总量:"+ccbPrices.size() + def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd')) + println "fjw总量:"+ccbPrices.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(ccbPrices.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i + def map = [:] + map.cityname = r.CITYNAME + map.distname = r.DISTNAME + map.commid = r.COMMID + map.commname = r.COMMNAME + map.year_month = r.YEAR_MONTH + map.calc_avg_price = r.CALC_AVG_PRICE + map.avg_price = r.AVG_PRICE + // 计算类型,1:人工估值 2:样本计算 3:神经网络 + if(r.CALC_TYPE){ + if(r.CALC_TYPE == 1){ + map.calc_type = "人工估值" + }else if(r.CALC_TYPE == 2){ + map.calc_type = "样本计算" + }else if(r.CALC_TYPE == 3){ + map.calc_type = "神经网络" + } + } + allRows << map + }); + return allRows + } + + + def getCommunityBasePriceGjByCommid(cityid, commid, yearMonth) { + def sql = "select AVG_PRICE, CALC_TYPE,CONFIDENCE from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} and commid='${commid}'"; + def map = [:] + sqlInstance236.eachRow(sql, { r -> + map.avg_price = r.AVG_PRICE + // 计算类型,1:人工估值 2:样本计算 3:神经网络 + if(r.CALC_TYPE){ + if(r.CALC_TYPE == 1){ + map.calc_type = "人工估值" + }else if(r.CALC_TYPE == 2){ + map.calc_type = "样本计算" + }else if(r.CALC_TYPE == 3){ + map.calc_type = "神经网络" + } + } + map.credit = r.CONFIDENCE + }); + return map + } + + + def batchCreateFile(ret, fileName, headers, charset){ + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); +// sb.append(headers.join("@|@")) +// sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row.toArray(), "@|@")) + sb.append(System.getProperty("line.separator")); + + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 08e507af9..ba2d7db15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_252; def rows = [] for (def city : citys) { @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-06-1", "2022-07-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) @@ -147,7 +147,7 @@ class GuiDingPriceBatch extends BaseBatch{ ret << dist._id.toString() ret << dist.name ret << cityCodeDatas.get(dist.city) - ret << dist.city + ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 3ac7f9a28..ed95e4cc6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -24,7 +24,7 @@ class StagingDistrictBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for(def city : citys){ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index bef978957..31ae30039 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,8 +30,8 @@ import java.util.regex.*; //def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", -// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") //def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -45,862 +45,996 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def url = "https://sh.anjuke.com" -//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") -//println s - -//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") -//for(def d : districts){ -// println d.name -//} - -//def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) -//for(def d : dists){ -// def o = CommonDatautils.findDistrictById(d._id.toString()) -// println o._id.toString() + " "+o.name -//} - -//def citys = CcbCommonUtils.citys; -//def rows = [] +////def url = "https://sh.anjuke.com" +////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") +////println s // -//for(def city : citys) { -// def data = CcbCommonUtils.findCityCodeValue() -// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +////def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") +////for(def d : districts){ +//// println d.name +////} // -// def count = 0 -// sqlInstance.execute(sql,{ -// count = it.count -// }); -// println city +" "+count -//} - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) -//} -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) -// } -//},2) - - - - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" - -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// def map = [:] -// map.put("land_area",d['D']) -// map.put("build_area",d['F']) -// map.put("greeningrate_area",d['H']) -// map.put("parkingcnt_area",d['J']) -// map.put("zts_area",d['L']) -// map.put("buildyear_area",d['N']) -// map.put("plotratio",d['P']) +////def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) +////for(def d : dists){ +//// def o = CommonDatautils.findDistrictById(d._id.toString()) +//// println o._id.toString() + " "+o.name +////} // -// dealMap.put(d['A'], map) -//} - -//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -////小区量 -//ExcelUtils.multiThreadingModify(fileName, { d -> -// def city = d['B'] -// if (city) { -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['P'] = total -// } -//},0) - -////小区字段覆盖率 -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def city = d['A'] -// if(city){ -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['AA'] = total -// -// def a = findEstateType(cityid, sqlInstance) -// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -// -// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -// -// def c = findFiledCount(cityid, 'address', sqlInstance) -// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -// -// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -// -// def e = findFiledCount(cityid, 'landarea', sqlInstance) -// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -// -// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -// -// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -// -// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -// -// def i = findFiledCount(cityid, 'latitude', sqlInstance) -// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -// -// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) -// println d -// } -//},1) - -//def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" -//def datas =ExcelUtils.read(fileName,0,1) -//def index = 0 -//def rows = [] -//for(def d : datas) { -//// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// if (count == 0) { -// index++ -// rows << d -// println index +" "+d -// } -//// if(rows.size()>20){ -//// break -//// } -//// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") -//// if(dists.size()==1){ -//// def dist = dists[0] -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// }else{ -//// for(def r :dists){ -//// if(d['B'].contains(r.region)){ -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// } -//// } -//// } -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") -//println "共查到:"+index - - -def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ - def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ - def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" - def count = 0 - sqlInstance.eachRow(sql,{r-> - if(r.dealunitprice == price){ - count++ - }else{ - println r.dealunitprice +" price:"+price - } - - }); - return count -} - - -def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ - def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") - def q = new BasicDBObject() - q.put("contractTime", date); -// q.put("area", new BasicDBObject('$gte', 20)) - q.put("fjCity",city); - q.put("fjRegion",region); - q.put("fjName",name); - q.put("area",area); - q.put("price", price) - def count = youda_esfhouse_coll.count(q) - return count; -} - - -//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202203) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['X'] = oldcount -// def casecount = testMap.get(d['C']) -// d['Y'] = casecount -// println d -// } -//},0) -//def rows = findTradePpeData(sqlInstance236) -//// rows.unique() -//def map=findTradeDetail(sqlInstance236,null) -//def datas=[] -//map.each{key,row-> -// if(!rows.contains(key)){ -// datas< -// def rows = findTradePpeData(sqlInstance236, 202203,code) -// def alltrades=findTradeDetail(sqlInstance236,code) -// alltrades.each{it-> -// def commid=it[0] -// if(!rows.contains(commid)){ -// datas< -// if(d['I'] && d['K']){ -// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") -// if(dist){ -// d['S'] = dist._id.toString() -// } -// } -//},0) - -//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> -// if(d['A']){ -// def obj = CommonDatautils.findDistrictById(d['A']) -// if(obj && obj.name.contains("一期")){ -// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) -// if(newName){ -// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") -// if(masterDist && masterDist.name_cat==0){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 9) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", obj._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// d['D'] = "主名不存在" -// } -// } -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> -// def flag = d['M'] -// if(flag==0.0){ -// if(d['D']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['D'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -// } -// -//},0) - -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } -// } - } -},0) - - -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } -// } - } -},0) - - - - -//def citys = CcbCommonUtils.citys -//def rets = [] -//for(def city : citys){ -// def count = CommonDatautils.findSeaweeds(city, "district", 1) -// rets << [city, count] -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") - -//def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] -//for(def city : citys){ -// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) -// def index = 0 -// for(def dist : districts){ -// if(dist.name_cat && dist.name_cat==1){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 0) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// } -// } -// println city +" "+index -//} - - -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> -// if(d['C']){ -//// def udata = new BasicDBObject() -//// udata.put("name_cat", 0) -//// udata.put("m_date",new Date()) +////def citys = CcbCommonUtils.citys; +////def rows = [] //// -//// def q = new BasicDBObject() -//// q.put("_id", new ObjectId(d['C'])) -//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// def count = findDistCount(d['C'], sqlInstance); -// if(count==0){ -// println d['C'] -// } -// } -//},0) - - +////for(def city : citys) { +//// def data = CcbCommonUtils.findCityCodeValue() +//// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +//// +//// def count = 0 +//// sqlInstance.execute(sql,{ +//// count = it.count +//// }); +//// println city +" "+count +////} +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(d['C']){ +//// d['Q'] = dealMap.get(d['C']) +//// } +////},2) +// +// +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" +// +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// def map = [:] +//// map.put("land_area",d['D']) +//// map.put("build_area",d['F']) +//// map.put("greeningrate_area",d['H']) +//// map.put("parkingcnt_area",d['J']) +//// map.put("zts_area",d['L']) +//// map.put("buildyear_area",d['N']) +//// map.put("plotratio",d['P']) +//// +//// dealMap.put(d['A'], map) +////} +// +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//////小区量 +////ExcelUtils.multiThreadingModify(fileName, { d -> +//// def city = d['B'] +//// if (city) { +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['P'] = total +//// } +////},0) +// +//////小区字段覆盖率 +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def city = d['A'] +//// if(city){ +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['AA'] = total +//// +//// def a = findEstateType(cityid, sqlInstance) +//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +//// +//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +//// +//// def c = findFiledCount(cityid, 'address', sqlInstance) +//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +//// +//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +//// +//// def e = findFiledCount(cityid, 'landarea', sqlInstance) +//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +//// +//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +//// +//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +//// +//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +//// +//// def i = findFiledCount(cityid, 'latitude', sqlInstance) +//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +//// +//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +//// println d +//// } +////},1) +// +////def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" +////def datas =ExcelUtils.read(fileName,0,1) +////def index = 0 +////def rows = [] +////for(def d : datas) { +////// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// if (count == 0) { +//// index++ +//// rows << d +//// println index +" "+d +//// } +////// if(rows.size()>20){ +////// break +////// } +////// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") +////// if(dists.size()==1){ +////// def dist = dists[0] +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// }else{ +////// for(def r :dists){ +////// if(d['B'].contains(r.region)){ +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// } +////// } +////// } +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") +////println "共查到:"+index +// +// //def index = 0 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ +////def commIdThirdMap = findAllThird(202207, sqlInstance236); +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +////cityCodeDatas.each{city,cityId-> +//// +//// deleteThirdByCityId(cityId, 202207, sqlInstance236) +//// println city + "完成" +////} // -// def alias = [] -// if(d['H'] && d['H']!=dist.name){ -// alias << d['H'] -// if(d['H'].contains("·")){ -// alias << d['H'].replace("·", "") -// } -// if(dist.alias){ -// for(def aliasName : dist.alias){ -// if(!alias.contains(aliasName)){ -// alias << aliasName -// } -// } -// } -// } // -// def address = null -// if(d['I']){ -// address = AddressUtil.formatAddress(dist.city, d['I']) -// } -// -// Long lng = null -// Long lat = null -// Long lat2 = null -// Long lng2 = null -// if(d['J']){ -// lng = d['J'].toString().toDouble() * 10000000000l; -// lat = d['K'].toString().toDouble() * 10000000000l; -// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) -// if(json != null && json.containsKey("x")) { -// lat2 = json.get("y") * 10000000000l -// lng2 = json.get("x") * 10000000000l -// } -// } -// -// def region = null -// if(dist.region!=d['F']){ -// region = dist.region -// } -// -// def udata = new BasicDBObject() -// if(region){ -// udata.put("region", region) -// } -// -// if(address){ -// udata.put("address", address) -// } -// if(lat2){ -// udata.put("lng", lng) -// udata.put("lat", lat) -// udata.put("lng2", lng2) -// udata.put("lat2", lat2) -// } -// -// if(alias){ -// udata.put("alias", alias) -// } -// if(udata){ -// index++ -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) -// println index+ " "+dist.city +" "+dist.name+" "+udata -// } -// -// } +//def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) +//for(line in lines) { +// def arrs = line.toString().split("\\|@\\|") +// if(arrs){ +// def sql = "delete from guided_price_gj where id='${arrs[0]}'" +// sqlInstance236.executeUpdate(sql) // } +//} // -//},0) - - - -//def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) -// if(obj && obj.alias){ -// d['H'] = obj.alias.join(" ") -// } -//// def citycode = d['A'] -//// cityCodeDataMap.each {k,v-> -//// if(citycode.contains(v)){ -//// d['D'] = k +//def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) +//} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> +//// if (d['C']) { +//// index ++T +//// def cityId = cityCodeDatas.get(d['A']) +//// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) +//// if(sources.size()>1){ +//// if(sources.contains("安居客") && sources.contains("58同城")){ +//// sources.remove("58同城") +//// }else if(sources.contains("链家") && sources.contains("贝壳")){ +//// sources.remove("贝壳") +//// } +//// } +//// if(sources.size()==1){ +//// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId +//// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) //// } //// } -//},0) - - - - -//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" -//def datas = ExcelUtils.read(fileName, 0, 1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) +////},0) +// +// +//def findAllThird(price_date, def sqlInstance236){ +// def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" +// def map = [:] +// sqlInstance236.eachRow(sql,{r-> +// def sources = map.get(r.commid) +// if(!sources){ +// sources = [] +// map.put(r.commid, sources) +// } +// sources << r.data_source +// }); +// return map //} -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> -// if(d['A']){ -// d['W'] = dealMap.get(d['A']) -// } -//},0) - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def deleteIndex = 0 -//ExcelUtils.modify(fileName,{d-> -// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ -// def dist = CommonDatautils.findDistrictById(d['D']) -// def city = dist.city -// def region = RegionData.formatRegion(city, d['Q']) -// if(region && dist.region!=region){ -// println city +" new:"+region +" old:"+d['C'] -// def udata = new BasicDBObject() -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// deleteIndex++ -// udata.put("status", 1) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// }else{ -// updateIndex++ -// udata.put("region", region) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// } -// }else{ -// println d['D']+" 不存在" -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// if(d['K']){ -// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) -// def address = null -// if(formatAddress && formatAddress.endsWith("号")) { -// address = formatAddress -// } // // -// def data = new BasicDBObject(); -// data.put("m_date", new Date()) -// if(d['M']){ -// long glat = d['M'].toString().toDouble() * 10000000000L -// long glng = d['L'].toString().toDouble() * 10000000000L -// data.put("lat",glat) -// data.put("lng",glng) -// -// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) -// if(baidu){ -// long blng = baidu.x * 10000000000L -// long blat = baidu.y * 10000000000L -// data.put("lat2",blat) -// data.put("lng2",blng) -// } -// -// } -// -// if(address){ -// data.put("address", address) -// } -// index++ -// println index +" "+d['B'] + " "+data -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) -// } -// } -// } -//}) - - -def findDealCount(sqlInstance, def startDate, def endDate){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate - map.put(r.cityname, r.count) - }); - return map -} - - -def findCaluateCityPPE(def sqlInstance, def yearMonth){ - def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; - def map = [:] - sqlInstance.eachRow(sql,{r-> - map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) - }); - return map -} - - -def findTradePpeData(def sqlInstance){ - def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; - def rows = new HashSet() - sqlInstance.eachRow(sql,{r-> - rows< - def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate - def row=[] - row< - count = r.count - }); - return count -} - -def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - - -def findDistCount(commid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def delIndex = 0 -//def cityMap = [:] -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(!d['R']){ -// -// def obj = CommonDatautils.findDistrictById(d['D']) -// if(obj){ -// def ranges = cityMap.get(obj.city) -// if(!ranges){ -// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) -// ranges = [] -// for(def r : dbcursor){ -// ranges << r -// } -// cityMap.put(obj.city, ranges) -// } -// -// if(ranges){ -// def lng = obj.lng/10000000000.0 -// def lat = obj.lat/10000000000.0 -// -// def region = null -// for(def r : ranges){ -// try{ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// def regionStr = r.region -// region = RegionData.formatRegion(obj.city, regionStr) -// break -// } -// }catch(Exception e){ -// println e.getMessage() -// } -// -// } -// if(region && region!=obj.region){ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name -// }else if(region && region==obj.region){ -// println "[same] "+obj.city +" "+obj.name -// }else{ -// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// delIndex++ -// println delIndex +" [delete] "+obj.city +" "+obj.name -// } -// }else{ -// println obj.city +" 没有此城市边框信息" -// } -// -// }else{ -// println d['D']+" 没找到该小区" -// } -// -// } -//},0); - - - -//def city = "杭州" -//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) -//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) -//def ranges = [] -//for(def d : dbcursor){ -// ranges << d +//def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) //} -//def index =0 -//def updateIndex = 0 -//def delIndex = 0 -//for(def dist : dists){ -//// if(dist.region=="江干"){ -// def lat = dist.lat/10000000000.0 -// def lng = dist.lng/10000000000.0 // -// def regionStr =null -// def region = null -// for(def r : ranges){ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// regionStr = r.region -// region = RegionData.formatRegion(dist.city, regionStr) -// break -// } -// } -// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) -// if(regionStr && region!=dist.region){ +//def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// def sources = [] +// sqlInstance236.eachRow(sql,{r-> +// sources << r.data_source +// }); +// return sources +//} // -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// delIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// println delIndex+" [del]"+region +" "+dist.name -// }else{ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name -// } +// +//def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ +// def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ +// def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// if(r.dealunitprice == price){ +// count++ // }else{ -// index++ -// println index+" [没找到]"+dist.name +// println r.dealunitprice +" price:"+price // } +// +// }); +// return count +//} +// +// +//def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ +// def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") +// def q = new BasicDBObject() +// q.put("contractTime", date); +//// q.put("area", new BasicDBObject('$gte', 20)) +// q.put("fjCity",city); +// q.put("fjRegion",region); +// q.put("fjName",name); +// q.put("area",area); +// q.put("price", price) +// def count = youda_esfhouse_coll.count(q) +// return count; +//} +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +////def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") +////def testMap = findCaluateCityPPE(sqlInstance236, 202203) +////ExcelUtils.modify(fileName,{d-> +//// if(d['C']){ +//// def oldcount = dealMap.get(d['C']) +//// d['X'] = oldcount +//// def casecount = testMap.get(d['C']) +//// d['Y'] = casecount +//// println d //// } -//} - - - - - - -def isInArea(lng, lat, points){ - List pointList = InsidePolygon.createPoint2(points); - //Point point = new Point(lng, lat); - //def flag = InsidePolygon.aboutZone(point, pointList) - def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) - return flag; -} - -def getStreetInfo(def city, def glocation){ - def township = null - if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) - if(jsonObject && jsonObject.regeocode) { - def addressComponent = jsonObject.regeocode.addressComponent - if(addressComponent && addressComponent.city.contains(city)){ - township = addressComponent.township - } - } - } - return township; -} - - -//def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//def oracleMap = [:] -//for(def d : oracleDates){ -// oracleMap.put(d['A'], d['B']) +////},0) +////def rows = findTradePpeData(sqlInstance236) +////// rows.unique() +////def map=findTradeDetail(sqlInstance236,null) +////def datas=[] +////map.each{key,row-> +//// if(!rows.contains(key)){ +//// datas< +//// def rows = findTradePpeData(sqlInstance236, 202203,code) +//// def alltrades=findTradeDetail(sqlInstance236,code) +//// alltrades.each{it-> +//// def commid=it[0] +//// if(!rows.contains(commid)){ +//// datas< +//// if(d['I'] && d['K']){ +//// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") +//// if(dist){ +//// d['S'] = dist._id.toString() +//// } +//// } +////},0) +// +////0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> +//// if(d['A']){ +//// def obj = CommonDatautils.findDistrictById(d['A']) +//// if(obj && obj.name.contains("一期")){ +//// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) +//// if(newName){ +//// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") +//// if(masterDist && masterDist.name_cat==0){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 9) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", obj._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// d['D'] = "主名不存在" +//// } +//// } +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> +//// def flag = d['M'] +//// if(flag==0.0){ +//// if(d['D']){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", new ObjectId(d['D'])) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// println index +//// } +//// } +//// +////},0) +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +// +// +////def citys = CcbCommonUtils.citys +////def rets = [] +////for(def city : citys){ +//// def count = CommonDatautils.findSeaweeds(city, "district", 1) +//// rets << [city, count] +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") +// +////def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] +////for(def city : citys){ +//// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) +//// def index = 0 +//// for(def dist : districts){ +//// if(dist.name_cat && dist.name_cat==1){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 0) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// } +//// } +//// println city +" "+index +////} +// +// +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> +//// if(d['C']){ +////// def udata = new BasicDBObject() +////// udata.put("name_cat", 0) +////// udata.put("m_date",new Date()) +////// +////// def q = new BasicDBObject() +////// q.put("_id", new ObjectId(d['C'])) +////// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// def count = findDistCount(d['C'], sqlInstance); +//// if(count==0){ +//// println d['C'] +//// } +//// } +////},0) +// +// +////def index = 0 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// +//// def alias = [] +//// if(d['H'] && d['H']!=dist.name){ +//// alias << d['H'] +//// if(d['H'].contains("·")){ +//// alias << d['H'].replace("·", "") +//// } +//// if(dist.alias){ +//// for(def aliasName : dist.alias){ +//// if(!alias.contains(aliasName)){ +//// alias << aliasName +//// } +//// } +//// } +//// } +//// +//// def address = null +//// if(d['I']){ +//// address = AddressUtil.formatAddress(dist.city, d['I']) +//// } +//// +//// Long lng = null +//// Long lat = null +//// Long lat2 = null +//// Long lng2 = null +//// if(d['J']){ +//// lng = d['J'].toString().toDouble() * 10000000000l; +//// lat = d['K'].toString().toDouble() * 10000000000l; +//// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +//// if(json != null && json.containsKey("x")) { +//// lat2 = json.get("y") * 10000000000l +//// lng2 = json.get("x") * 10000000000l +//// } +//// } +//// +//// def region = null +//// if(dist.region!=d['F']){ +//// region = dist.region +//// } +//// +//// def udata = new BasicDBObject() +//// if(region){ +//// udata.put("region", region) +//// } +//// +//// if(address){ +//// udata.put("address", address) +//// } +//// if(lat2){ +//// udata.put("lng", lng) +//// udata.put("lat", lat) +//// udata.put("lng2", lng2) +//// udata.put("lat2", lat2) +//// } +//// +//// if(alias){ +//// udata.put("alias", alias) +//// } +//// if(udata){ +//// index++ +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +//// println index+ " "+dist.city +" "+dist.name+" "+udata +//// } +//// +//// } +//// } +//// +////},0) +// +// +// +////def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) +//// if(obj && obj.alias){ +//// d['H'] = obj.alias.join(" ") +//// } +////// def citycode = d['A'] +////// cityCodeDataMap.each {k,v-> +////// if(citycode.contains(v)){ +////// d['D'] = k +////// } +////// } +////},0) +// +// +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" +////def datas = ExcelUtils.read(fileName, 0, 1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> +//// if(d['A']){ +//// d['W'] = dealMap.get(d['A']) +//// } +////},0) +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def deleteIndex = 0 +////ExcelUtils.modify(fileName,{d-> +//// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ +//// def dist = CommonDatautils.findDistrictById(d['D']) +//// def city = dist.city +//// def region = RegionData.formatRegion(city, d['Q']) +//// if(region && dist.region!=region){ +//// println city +" new:"+region +" old:"+d['C'] +//// def udata = new BasicDBObject() +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// deleteIndex++ +//// udata.put("status", 1) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// }else{ +//// updateIndex++ +//// udata.put("region", region) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// } +//// }else{ +//// println d['D']+" 不存在" +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// if(d['K']){ +//// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) +//// def address = null +//// if(formatAddress && formatAddress.endsWith("号")) { +//// address = formatAddress +//// } +//// +//// +//// def data = new BasicDBObject(); +//// data.put("m_date", new Date()) +//// if(d['M']){ +//// long glat = d['M'].toString().toDouble() * 10000000000L +//// long glng = d['L'].toString().toDouble() * 10000000000L +//// data.put("lat",glat) +//// data.put("lng",glng) +//// +//// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) +//// if(baidu){ +//// long blng = baidu.x * 10000000000L +//// long blat = baidu.y * 10000000000L +//// data.put("lat2",blat) +//// data.put("lng2",blng) +//// } +//// +//// } +//// +//// if(address){ +//// data.put("address", address) +//// } +//// index++ +//// println index +" "+d['B'] + " "+data +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) +//// } +//// } +//// } +////}) +// +// +//def findDealCount(sqlInstance, def startDate, def endDate){ +// def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate +// map.put(r.cityname, r.count) +// }); +// return map //} // -//def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) -//def chMap = [:] -//for(def h : chDates){ -// chMap.put(h['A'].toString().replace(".0",""), h['B']) -//} - -//def citys = CcbCommonUtils.citys; -//def map = [:] -//def rets = [] -//for(def c : citys){ -// def citycode = cityCodeDataMap.get(c) -// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") - -//def weekend = DateUtils.getFirstDayOfWeek(new Date()) -//def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); -//def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) -//def rets = [] // -//for(def dist : districts){ -// def row = [] -// row << dist.get("_id").toString() -// row << dist.city -// row << dist.region -// row << dist.name -// if(dist.lng){ -// row << dist.lng/10000000000.0 -// row << dist.lat/10000000000.0 -// }else{ -// row << "" -// row << "" +//def findCaluateCityPPE(def sqlInstance, def yearMonth){ +// def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; +// def map = [:] +// sqlInstance.eachRow(sql,{r-> +// map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) +// }); +// return map +//} +// +// +//def findTradePpeData(def sqlInstance){ +// def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; +// def rows = new HashSet() +// sqlInstance.eachRow(sql,{r-> +// rows< +// def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate +// def row=[] +// row< +// count = r.count +// }); +// return count +//} +// +//def findCount(cityid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +// +//def findDistCount(commid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findFiledCount(cityid, field, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def delIndex = 0 +////def cityMap = [:] +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(!d['R']){ +//// +//// def obj = CommonDatautils.findDistrictById(d['D']) +//// if(obj){ +//// def ranges = cityMap.get(obj.city) +//// if(!ranges){ +//// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) +//// ranges = [] +//// for(def r : dbcursor){ +//// ranges << r +//// } +//// cityMap.put(obj.city, ranges) +//// } +//// +//// if(ranges){ +//// def lng = obj.lng/10000000000.0 +//// def lat = obj.lat/10000000000.0 +//// +//// def region = null +//// for(def r : ranges){ +//// try{ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// def regionStr = r.region +//// region = RegionData.formatRegion(obj.city, regionStr) +//// break +//// } +//// }catch(Exception e){ +//// println e.getMessage() +//// } +//// +//// } +//// if(region && region!=obj.region){ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name +//// }else if(region && region==obj.region){ +//// println "[same] "+obj.city +" "+obj.name +//// }else{ +//// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// delIndex++ +//// println delIndex +" [delete] "+obj.city +" "+obj.name +//// } +//// }else{ +//// println obj.city +" 没有此城市边框信息" +//// } +//// +//// }else{ +//// println d['D']+" 没找到该小区" +//// } +//// +//// } +////},0); +// +// +// +////def city = "杭州" +////def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +////def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) +////def ranges = [] +////for(def d : dbcursor){ +//// ranges << d +////} +////def index =0 +////def updateIndex = 0 +////def delIndex = 0 +////for(def dist : dists){ +////// if(dist.region=="江干"){ +//// def lat = dist.lat/10000000000.0 +//// def lng = dist.lng/10000000000.0 +//// +//// def regionStr =null +//// def region = null +//// for(def r : ranges){ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// regionStr = r.region +//// region = RegionData.formatRegion(dist.city, regionStr) +//// break +//// } +//// } +//// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +//// if(regionStr && region!=dist.region){ +//// +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// delIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// println delIndex+" [del]"+region +" "+dist.name +//// }else{ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name +//// } +//// }else{ +//// index++ +//// println index+" [没找到]"+dist.name +//// } +////// } +////} +// +// +// +// +// +// +//def isInArea(lng, lat, points){ +// List pointList = InsidePolygon.createPoint2(points); +// //Point point = new Point(lng, lat); +// //def flag = InsidePolygon.aboutZone(point, pointList) +// def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) +// return flag; +//} +// +//def getStreetInfo(def city, def glocation){ +// def township = null +// if(glocation){ +// def jsonObject = GaoDeApiUtils.geocode(glocation) +// if(jsonObject && jsonObject.regeocode) { +// def addressComponent = jsonObject.regeocode.addressComponent +// if(addressComponent && addressComponent.city.contains(city)){ +// township = addressComponent.township +// } +// } // } -// -// row << statMap.get(dist.region +"##"+dist.name) -// rets << row +// return township; //} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") - -//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) -//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) // +// +////def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////def oracleMap = [:] +////for(def d : oracleDates){ +//// oracleMap.put(d['A'], d['B']) +////} +//// +////def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) +////def chMap = [:] +////for(def h : chDates){ +//// chMap.put(h['A'].toString().replace(".0",""), h['B']) +////} +// +////def citys = CcbCommonUtils.citys; +////def map = [:] +////def rets = [] +////for(def c : citys){ +//// def citycode = cityCodeDataMap.get(c) +//// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") +// +////def weekend = DateUtils.getFirstDayOfWeek(new Date()) +////def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); +////def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) +////def rets = [] +//// +////for(def dist : districts){ +//// def row = [] +//// row << dist.get("_id").toString() +//// row << dist.city +//// row << dist.region +//// row << dist.name +//// if(dist.lng){ +//// row << dist.lng/10000000000.0 +//// row << dist.lat/10000000000.0 +//// }else{ +//// row << "" +//// row << "" +//// } +//// +//// row << statMap.get(dist.region +"##"+dist.name) +//// rets << row +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") +// +////def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +////def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) +//// +////def keys = [] +////for(def d : datas2){ +//// keys << d['A']+"##"+d['C']+"##"+d['B'] +////} +//// +////def index = 0 +////for(def d : datas1){ +//// def k = d['A']+"##"+d['C']+"##"+d['B'] +//// if(!keys.contains(k)){ +//// index ++ +//// println index +" "+k +//// } +////} +// +// +// +// +//def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ +// def q = new BasicDBObject() +// q.put("city", city) +// q.put("weekend", weekend) +// def dbcursor = DIST_SELL_STAT.find(q) +// def map = new ConcurrentHashMap() +// for(def d : dbcursor){ +// map.put(d.region+"##"+d.district_name, d.avg_price) +// } +// return map +//} + + + +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx" +//def datas = ExcelUtils.read(fileName, 0, 1) //def keys = [] -//for(def d : datas2){ -// keys << d['A']+"##"+d['C']+"##"+d['B'] +//for(def d : datas){ +// def key = d['A']+"##"+d['B']+"##"+d['C'] +// keys << key //} // -//def index = 0 +//def fileName1 = "D:\\数据\\建行\\房产估值\\指导价\\moban_ccb_0922.xlsx" +//def datas1 = ExcelUtils.read(fileName1, 0, 1) +//def keys1 = [] //for(def d : datas1){ -// def k = d['A']+"##"+d['C']+"##"+d['B'] +// def key = d['A']+"##"+d['B']+"##"+d['C'] +// keys1 << key +//} +// +//for(def k : keys1){ // if(!keys.contains(k)){ -// index ++ -// println index +" "+k +// println k // } //} +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx" -def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ - def q = new BasicDBObject() - q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) - def map = new ConcurrentHashMap() - for(def d : dbcursor){ - map.put(d.region+"##"+d.district_name, d.avg_price) - } - return map -} +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +// def region = RegionData.formatRegion(d['A'], d['B']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") +// if(dist){ +// d['E'] = dist.region +// d['F'] = dist._id.toString() +// } +// } +//// if(!d['Q']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +////// +//// if(dist){ +//// d['H'] = dist.region +////// d['G'] = dist.name +//// d['I'] = dist.status +//// } +//// } +// +// } +//},0) + + + +//ExcelUtils.modify(fileName, {d-> +// if(d['A']){ +// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" +// println sql +// sqlInstance236.executeUpdate(sql) +// } +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 719250e09..6b1009e6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202206//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202204 - def endDate = 202206 - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202107 + def endDate = 202109 +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index ee37a0616..c2f488b5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -73,15 +73,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_92; def rets = [] - rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] - def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] + rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-04-01" - def date = "2022-05-01" - def endDate = "2022-06-01" - def newDate = "2022-07-01" + def startDate = "2022-07-01" + def date = "2022-08-01" + def endDate = "2022-09-01" + def newDate = "2022-10-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\${newDate}成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbRegionCode.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbRegionCode.groovy new file mode 100644 index 000000000..b2541547c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbRegionCode.groovy @@ -0,0 +1,26 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + + +DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") + +def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx", 0, 1) +for(def d : datas){ + def dbo = new BasicDBObject() + dbo.put("province",d['C']) + dbo.put("city", d['D']) + dbo.put("district", d['G']) + dbo.put("city_code", d['E'].toInteger()) + dbo.put("district_code",d['F'].toInteger()) + dbo.put("fjw_city", d['A']) + dbo.put("fjw_district", d['B']) + dbo.put("c_date", new Date()) + FANGJIA_AREA_CODE.save(dbo) +} + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index be808deaa..b8116db22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_all; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { @@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // }) // } -// addBigdataDistrictFJW() + addBigdataDistrictFJW() - def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); - - def index = 0 - for (def city : citys) { - index++ - def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); // -// for(int i=0;i<10;i++){ -// if(districts.size()==0){ -// Thread.sleep(1000*60) -// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -// }else{ -// break +// def index = 0 +// for (def city : citys) { +// index++ +// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +//// +//// for(int i=0;i<10;i++){ +//// if(districts.size()==0){ +//// Thread.sleep(1000*60) +//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +//// }else{ +//// break +//// } +//// } +// +// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) +// +// for(def dist : districts){ +// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) +// if(obj){ +// if(obj.alias && obj.alias.size()>0){ +// dist.commalias = obj.alias.get(0) +// }else{ +// dist.commalias = "" +// } +// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) +// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) +// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); +// dist.houseuse = "11" +// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) +// dist.compid = new ObjectId() //物业公司id +// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) // } // } - - def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) - - for(def dist : districts){ - def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) - if(obj){ - if(obj.alias && obj.alias.size()>0){ - dist.commalias = obj.alias.get(0) - }else{ - dist.commalias = "" - } - dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) - dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) - dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); - dist.houseuse = "11" - dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) - dist.compid = new ObjectId() //物业公司id - dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) - } - } - println index +" "+city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 4.小区建筑年代表" - addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 5.小区费用表" - addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 6.小区房屋用途表" - addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) - println index +" "+ city + " 7.地理位置表" - -// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 - - } +// println index +" "+city +" 小区加载完成 "+districts.size() +// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 1.小区表完成" +// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 2.小区建筑结构表" +// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 3.小区房屋类型表" +// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 4.小区建筑年代表" +// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 5.小区费用表" +// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 6.小区房屋用途表" +// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) +// println index +" "+ city + " 7.地理位置表" +// +//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 +// +// } } def addBigdataCommunityGJRH(def districts, def cityid){ @@ -557,14 +557,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch { def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){ def rets = [] + def num = 0 for(def d : districts) { + num = num + 1 def ret = [] + def detailMap = citySeqnoMap.get(d.cityid) def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap); ret << d.cityid ret << "5" ret << d.commid ret << "1" - ret << seqno + if(detailMap && seqno){ + ret << seqno + }else{ + ret << num + } ret << d.lng ret << d.lat ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index e50059409..386f96f4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -22,25 +22,29 @@ class ExportDistrictRongHe extends BaseBatch{ runBatch(new ExportDistrictRongHe(), args); } + def citys = [] + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + @Override void process(Object args) { + citys = CcbCommonUtils.getRunningCitys(args) def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] - //按天导(每天一个文件夹) + //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH() - exportBigdataBuildingFJW() - exportBigdataCommfeeGj() + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - exportBigdataCommbuildstruGJ() - exportBigdataCommhousetypeGJ() - exportBigdataCommbuildyearGj() - exportBigdataCommhouseuseGJ() - exportBigdataPositionGJ() - exportBigdataDistrictFJW() - exportBigdataAliasFJW() + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// exportBigdataBuildprojFjw() exportBigdataCommlayoutpicFjw() @@ -59,14 +63,14 @@ class ExportDistrictRongHe extends BaseBatch{ // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // // -//// //检查ok文件是否上传 +//// //妫�鏌k鏂囦欢鏄惁涓婁紶 //// exportAllBigdataDealFJW() //// for(def city : citys) { //// def cityCode = cityCodeDataMap.get(city) //// -// //按天导(一个文件夹下) +// //鎸夊ぉ瀵�(涓�涓枃浠跺す涓�) // //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// //按天导(每天一个文件夹) +// //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� // exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) // //// exportBigdataCommunityGJRH(cityCode) @@ -94,261 +98,308 @@ class ExportDistrictRongHe extends BaseBatch{ // exportOk() // } - def exportBigdataCommunityGJRH(cityCode){ - def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMMNAME - data << r.COMMALIAS - data << r.DISTID - data << r.STREET - data << r.STREETNO - data << r.ADDRESS - data << r.LANDAREA - data << r.BUILDAREA - data << r.COMMTYPE - data << r.IFNEW - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << '' - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + def exportBigdataCommunityGJRH(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" + def index = 0 + for(def city : citys){ + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMMNAME + data << r.COMMALIAS + data << r.DISTID + data << r.STREET + data << r.STREETNO + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.COMMTYPE + data << r.IFNEW + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << '' + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + + } + println "BIGDATA_COMMUNITY_FJW 鏁版嵁鏂囦欢杈撳嚭瀹屾垚......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" + println "BIGDATA_COMMUNITY_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommbuildstruGJ(){ - def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDSTRU - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" + println "BIGDATA_COMMBUILDSTRU_FJW 涓婁紶Ftp瀹屾垚......" } - def exportBigdataCommhousetypeGJ(cityCode){ - def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' + def exportBigdataCommhousetypeGJ(){ + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" + println "BIGDATA_COMMHOUSETYPE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommbuildyearGj(){ - def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDYEAR - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" + println "BIGDATA_COMMBUILDYEAR_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommfeeGj(){ - def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMPID - data << r.FEE - data << r.HOUSEUSE - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.FEETYPE - data << r.NOTE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" + println "BIGDATA_COMMFEE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommhouseuseGJ(){ - def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSEUSE - data << r.HOUSECNT - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" + println "BIGDATA_COMMHOUSEUSE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataPositionGJ(){ - def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.OBJECTTYPE - data << r.OBJECTID - data << r.POSITIONTYPE - data << r.SEQNO - data << r.LONGITUDE - data << r.LATITUDE - data << r.ALTITUDE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_POSITION_FJW 上传Ftp完成......" + println "BIGDATA_POSITION_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataDistrictFJW(){ - def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" + println "BIGDATA_DISTRICT_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataAliasFJW(){ - def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.city_id - data << r.ca_id - data << r.address_id - data << r.address_name - data << r.status - data << r.exp_dt - data << r.version - data << r.is_key_name - data << r.data_source - data << r.description - data << r.alias_grade - data << r.addr_type_cd - data << r.ifexamine - data << r.ifdisplay - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "ADDRESS_ALIAS_FJW 上传Ftp完成......" + println "ADDRESS_ALIAS_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataDealFJW(dealdate){ @@ -368,7 +419,7 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -453,7 +504,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" + println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr } @@ -468,31 +519,36 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] - if(curDate.toString().endsWith("28")){ - if(new_dists){ - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + - "where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + - " distid not in ${newstr} ";//CITYID='${cityCode}' and - def rets1 = createDeal(sql1) - def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + - " where distid in ${newstr}" - def rets2 = createDeal(sql2) - allRows.addAll(rets1) - allRows.addAll(rets2) - }else{ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and - allRows = createDeal(sql) + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("28")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + "where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + + " distid not in ${newstr} ";//CITYID='${cityCode}' and + def rets1 = createDeal(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + " where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = createDeal(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";// + allRows = createDeal(sql) + } } } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${endDateStr}" } def createDeal(sql){ @@ -653,54 +709,68 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_DEAL_FJW 导出完成......" + println "BIGDATA_DEAL_FJW 瀵煎嚭瀹屾垚......" } - def exportBigdataBuildingFJW(cityCode){ - def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.BUILDALIAS - data << r.DISTID - data << r.COMMID - data << r.BUILDYEAR - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.FLOORCNT - data << r.UNITCNT - data << r.HOUSECNT - data << r.IFELEVATOR - data << r.ELEVATORCNT - data << r.ELEVATORHOUSE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.VALID_FLAG - allRows << data - }); + def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + def index = 0 + for(def city : citys) { + + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.BUILDALIAS + data << r.DISTID + data << r.COMMID + data << r.BUILDYEAR + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.FLOORCNT + data << r.UNITCNT + data << r.HOUSECNT + data << r.IFELEVATOR + data << r.ELEVATORCNT + data << r.ELEVATORHOUSE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.VALID_FLAG + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + println "BIGDATA_BUILDING_FJW 涓婁紶Ftp瀹屾垚......" } @@ -724,40 +794,40 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "EVALUATE_RATE_FJW 上传Ftp完成......" + println "EVALUATE_RATE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" + println "BIGDATA_BUILDPROJ_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" + println "BIGDATA_COMMLAYOUTPIC_FJW 涓婁紶Ftp瀹屾垚......" } def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + dataCodeFileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" } FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index fede45dfc..3be6ffa9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -70,7 +70,7 @@ class CommonDatautils { public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl) public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl) - + public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -136,6 +136,13 @@ class CommonDatautils { return cityInfos; } + + + public static def findAlCityDetail(city){ + DBObject q = new BasicDBObject(); + q.put("display", city) + return SERVER_AREA_COLL.findOne(q); + } public static def findAllCityByGrade(def city_grade){ DBObject q = new BasicDBObject(); @@ -466,6 +473,7 @@ class CommonDatautils { if(dbo.mapping_alias){ detailMap.mapping_alias = dbo.mapping_alias; } + detailMap.estate_type = dbo.estate_type2 } }catch(Exception e){ @@ -747,7 +755,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","森望芳草绿岸") +// q.put("name","湖畔佳苑") DBObject keys = new BasicDBObject(); if(fields){ @@ -2042,6 +2050,20 @@ class CommonDatautils { } + public static def queryAllDistrictPrice(startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put('weekend', startDate); + DBCursor cur = DIST_STAT_COLL.find(queryCondition) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def rMap = [:]; + while(cur.hasNext()) { + DBObject dbo = cur.next() + rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo) + } + return rMap; + } + + public static def queryDistrictPrices(city, startDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); @@ -3145,8 +3167,7 @@ class CommonDatautils { DBObject dbo; def prices = [] def index = 0 - while(cur.hasNext()) { - def obj = cur.next() + for(def obj : cur) { if(index == 0){ dbo = obj } @@ -3313,5 +3334,16 @@ class CommonDatautils { def count = HOUSE_LOUDONG_COLL.count(q) return count; } + + static def findCityInfo(city){ + def q = new BasicDBObject() + q.put("city", city) + def cur = CITY_INFO_COLL.find(q).sort(new BasicDBObject("weekend",-1)).limit(1) + while(cur.hasNext()) { + DBObject bsobj = cur.next() + return bsobj; + } + return null + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy new file mode 100644 index 000000000..534a844e7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.crawl + +import com.alibaba.fastjson.JSONArray +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang.StringUtils + +class SchoolLocation { + + DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018) + + static main(args) { + def task = new SchoolLocation() + task.run() + } + + def run(){ + def query = new BasicDBObject() + query.put("province","江苏") + //query.put("lat",null) + query.put("lat",new BasicDBObject('$exists',false)) + def cursor = school_parse_coll.find(query) + def index = 0 + for(def d : cursor){ + index++ + def datas = getSchoolLatAndLng(d.city, d.name, d.address, false) + if(datas){ + for(def ret : datas){ + def addr = d.address + if(d.address.contains(d.city+"市")){ + def cityStr = d.city+"市" + addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size()) + } + if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){ + def udata = new BasicDBObject() + udata.put("lat", ret.lat.toString().toDouble()) + udata.put("lng", ret.lng.toString().toDouble()) + udata.put("status", 0) + school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + println index +" "+udata + break; + } + } + } + } + } + + def getSchoolLatAndLng(city, name, address, isFullMatch){ + if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ + return null; + } + address = address.trim().replace(" ","").replace("?","").replace(" ","") + String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all" + + List poiList = null; + com.alibaba.fastjson.JSONObject result = null; + try { + HttpRequest request = new HttpRequest(); + Map params = new HashMap(); + String response = request.get(url, params, "UTF-8"); + com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response) + if (jsonResponse != null) { + JSONArray ja = jsonResponse.getJSONArray("pois"); + if(ja != null && ja.size() > 0) { + poiList = new ArrayList<>(); + for(int i=0;i + udate.put(k,v) + } + udate.removeField("_id") + udate.put("m_date", new Date()) + FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate)); + }else{ + FZ_DIST_COLL.save(d) + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + + } + + def getFzDistricts(def city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = [] + FZ_DIST_COLL.find(q).each{it-> + rets << it + } + return rets + } + def processAlias(alias, def addr_alias){ def newAlias = [] if(alias){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 276c86cec..45f3cbe35 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{ def charset = "GBK" public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } @@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{ return sb.toString() } + def getFzDistricts(def city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = [] + FZ_DIST_COLL.find(q).each{it-> + rets << it + } + return rets + } + def createDatas(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", null); - def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - districts.addAll(removeDists) +// def districts = CommonDatautils.findAllDistricts(city, "district", null); +// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); +// districts.addAll(removeDists) + def districts = getFzDistricts(city) def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{ curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } + }else{ + curPrice = specialDistsPriceMap.get(rMap.name) + if(curPrice){ + def randMarkup = nextDouble(0, 1) + curPrice = (int)(curPrice*randMarkup/100 + curPrice) + } } def m = [] - if(curPrice){ + if(curPrice && dbo){ m << dbo.get("_id").toString()//均价ID }else{ m << new ObjectId().toString() @@ -135,7 +154,11 @@ class DistrictPriceExportBatch extends BaseBatch{ m << rMap.city m << rMap.region m << rMap.name - m << curPrice + if(curPrice){ + m << curPrice + }else{ + m << 0 + } m << curDate.toInteger(); result << m } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 778312b6b..78537bc39 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -24,7 +24,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() - task.priceMarkupRun(); +// task.priceMarkupRun(); task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() @@ -88,14 +88,21 @@ class FzDataResultStat { lastPrice = lastObj[5] } def markup = null - if(list && lastPrice){ - markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + try{ + if(list && lastPrice && lastPrice.toInteger()>0){ + markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + } + }catch(Exception e){ + e.printStackTrace() } + def dist = CommonDatautils.findDistrictById(list[1]) def flag = "有效" - if(dist.status!=0){ + + if(dist && dist.status!=0){ flag = "失效" } + rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") @@ -131,7 +138,7 @@ class FzDataResultStat { def regionPriceTrendRun(){ def city = "福州" - def add = -6; + def add = -9; def months = [] while(add<0){ months << DateUtils.addMonth(new Date(), add) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy new file mode 100644 index 000000000..cae471308 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -0,0 +1,176 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.io.FileUtils + +class FzDistrictMatch { + + + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + + + static main(args) { + def task = new FzDistrictMatch() +// task.run(); +// task.addPrice() + task.addThirdPrice() + } + + def addThirdPrice(){ + def map = read("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv"); + ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d -> + if(d['G']){ + def priceMap = map.get(d['G']) + def prices = [] + if(priceMap && priceMap.zhugezhaofang && priceMap.zhugezhaofang.isNumber()){ + d['J'] = priceMap.zhugezhaofang + prices << priceMap.zhugezhaofang.toInteger() + } + + if(priceMap && priceMap.anjuke && priceMap.anjuke.isNumber()){ + d['K'] = priceMap.anjuke + prices << priceMap.anjuke.toInteger() + } + + if(priceMap && priceMap.soufang && priceMap.soufang.isNumber()) { + d['L'] = priceMap.soufang + prices << priceMap.soufang.toInteger() + } + + if(priceMap && priceMap.lianjia && priceMap.lianjia.isNumber()){ + d['M'] = priceMap.lianjia + prices << priceMap.lianjia.toInteger() + } + + if(priceMap && priceMap.cityhouse && priceMap.cityhouse.isNumber()){ + d['N'] = priceMap.cityhouse + prices << priceMap.cityhouse.toInteger() + } + + if(priceMap && priceMap.fangtan && priceMap.fangtan.isNumber()){ + d['O'] = priceMap.fangtan + prices << priceMap.fangtan.toInteger() + } + + if(priceMap && priceMap.fangdoudou && priceMap.fangdoudou.isNumber()){ + d['P'] = priceMap.fangdoudou + prices << priceMap.fangdoudou.toInteger() + } + if(priceMap && priceMap.maitian && priceMap.maitian.isNumber()){ + d['Q'] = priceMap.maitian + prices << priceMap.maitian.toInteger() + } + + if(priceMap && priceMap.get("58tc") && priceMap.get("58tc").isNumber()){ + d['R'] = priceMap.get("58tc") + prices << priceMap.get("58tc").toInteger() + } + if(prices){ + d['S'] = (int)(prices.sum()/prices.size()) + } + } + },0) + } + + def read(fileName){ + List list1= FileUtils.readLines(new File(fileName)) + def map1 = [:] + for(def str : list1){ + def s = str.split("@\\|@") + def key = s[2]+"##"+s[3]+"##"+s[4] + def priceMap = [:] + map1.put(s[1], priceMap) + + if(s.size()>=7 && s[6]){ + priceMap.put("zhugezhaofang", s[6]) + } + + if(s.size()>=8 && s[7]){ + priceMap.put("anjuke", s[7]) + } + + if(s.size()>=9 && s[8]){ + priceMap.put("soufang", s[8]) + } + + if(s.size()>=10 && s[9]){ + priceMap.put("lianjia", s[9]) + } + + if(s.size()>=11 && s[10]){ + priceMap.put("cityhouse", s[10]) + } + + if(s.size()>=12 && s[11]){ + priceMap.put("fangtan", s[11]) + } + + if(s.size()>=13 && s[12]){ + priceMap.put("fangdoudou", s[12]) + } + if(s.size()>=14 && s[13]){ + priceMap.put("maitian", s[13]) + } + + try{ + if(s.size()>=15 && s[14]){ + priceMap.put("58tc", s[14]) + } + }catch(Exception e){ + e.getMessage() + } + + } + return map1 + } + + def addPrice(){ + ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d -> + if(d['G']){ + def avg_price = findFzDistrictStat( d['G'],202207) + d['F'] = avg_price + } + },0); + } + + + def findFzDistrictStat(id,lastMonthDate){ + def q = new BasicDBObject() + q.put("comm_id", id) + q.put("month", lastMonthDate) + def dbcursor = FZ_DISTRICT_STAT_COLL.find(q) + for(def d : dbcursor){ + return d.avg_price + } + return null + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx",{d-> + def region = d['C'] + if(!d['G'] && region){ + def name = d['B'].toString().replace("(","").replace(")","") +// name = d['B'].toString().replace("(非钢混)","").replace("(钢混)","") + region = RegionData.formatRegion("福州", region) +// def dist = CommonDatautils.findDistrict("福州", region, name, "district") +// if(dist){ +// d['G'] = dist._id.toString() +// d['H'] = dist.name +// } + def jsonObject = OpenApiClient.addressMatch("福州", name, "property", false, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + d['H'] = jsonObject.get("name") + d['G'] = jsonObject.get("id") + d['J'] = jsonObject.get("district") + } + + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index b5aa0e611..32c4d1f34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{ // int endDate = 20191201 def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd") - // def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); - def endDate = "20220710" + def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); // QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 43e47801d..43203f42f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index 9e3a6dd7e..ac8449aa8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -75,81 +75,98 @@ class ThirdPriceExportBatch extends BaseBatch{ row << dist.region row << dist.name row << o.s_date + def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice) row << zgf_price if(zgf_price){ add = true; } -// if(o.zhugezhaofang_esf_price){ -// add = true; -// } + def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice) row << ajk_price if(ajk_price){ add = true; } -// if(o.anjuke_esf_price){ -// add = true; -// } def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice) row << sf_price if(sf_price){ add = true; } -// if(o.soufang_esf_price){ -// add = true; -// } + def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice) row << lj_price if(lj_price){ add = true; } -// if(o.lianjia_esf_price){ -// add = true; -// } def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice) row << city_price if(city_price){ add = true; } -// if(o.cityhouse_esf_price){ -// add = true; -// } def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice) row << ft_price if(ft_price){ add = true; } -// if(o.fangtan_esf_price){ -// add = true; -// } def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice) row < -// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) -// if(keys){ -// d['E'] = keys.size() +//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d-> +//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) +//// if(keys){ +//// d['G'] = keys.size() +//// } +// +//// def rets = findHousesByCity(d['A'], "sell", 20210701) +// if(d['A']){ +// def count = findHouseCount(d['A']) +// d['H'] = count // } - -// def rets = findHousesByCity(d['A'], "sell", 20210701) - if(d['A']){ - def count = findHouseCount(d['A']) - d['F'] = count - } - -}) +// +//}) def findHouseCount(city){ @@ -288,7 +288,7 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ //} // -//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"] + //def rows = [] //rows << ["城市","小区量", "别名量", "地址别名量"] //for(def c : citys){ @@ -305,7 +305,24 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ // } // rows << [c, districts.size(), alias_count, addr_alias_count] //} -// -// -// -//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果") +//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果") + +//区域级统计 +def city = "合肥" +def regions = CommonDatautils.getCityRegions(city) +def rows = [] +for(def region : regions){ + def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"]) + def alias_count = 0 + def addr_alias_count = 0 + for(def r : districts){ + if(r.alias && r.alias.size()>0){ + alias_count ++ + } + if(r.addr_alias && r.addr_alias.size()>0){ + addr_alias_count++ + } + } + rows << [region, districts.size(), alias_count, addr_alias_count] +} +ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 6ff2d9586..9f3670a44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -110,6 +110,7 @@ class FangjiaClientUtils { public static def saveToEsPaimai(def data){ String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data) HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr) +// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy new file mode 100644 index 000000000..fe2ab3332 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy @@ -0,0 +1,158 @@ +package com.fangjia.script.gj.dongguanbank + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import org.apache.lucene.queryparser.classic.QueryParser + +class DongGuanBankTest { + static main(args) { + def task = new DongGuanBankTest() + def path = "D:\\数据\\东莞银行\\东莞银行测试_输出单价(2).xlsx" +// task.run(path) +// task.addPrice(path) + task.addMinMaxPrice(path) + } + + def run(path){ + ExcelUtils.multiThreadingModify(path,{d-> + def city = d['A'].replace("市","") + if(city){ + def region = RegionData.formatRegion(city, d['B']) + def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C']) + if(results && results.precise>=0){ + d['H'] = results.city + d['I'] = results.district + d['J'] = results.name + d['Q'] = results.propertytype + } + } + + }) + } + + + def addPrice(path){ + ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d -> + if(d['H']){ + def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district") + def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + if(priceObj){ + d['M'] = priceObj.avg_price + d['N'] = (priceObj.avg_price * d['G']).toInteger() + d['O'] = ((int)(priceObj.avg_price/100))*100 + if(dist){ + d['R'] = dist.estate_type2 + } + if(priceObj.value_type){ + if(priceObj.value_type == 1){ + d['S'] = "指导价" + }else if(priceObj.value_type == 2){ + d['S'] = "自身样本" + }else if(priceObj.value_type == 3){ + d['S'] = "第三方" + }else if(priceObj.value_type == 4){ + d['S'] = "交易数据" + }else if(priceObj.value_type == 5){ + d['S'] = "周边楼盘" + }else if(priceObj.value_type == 6){ + d['S'] = "推导" + }else if(priceObj.value_type == 9){ + d['S'] = "神经网络" + }else if(priceObj.value_type == 10){ + d['S'] = "无方式" + } + } + } + + if(!d['P']){ + def alias = [] + if(!dist.alias){ + alias = [] + }else{ + alias = dist.alias + } + alias << dist.name + def map = findMaxAvgPrice(dist.city, dist.region, alias) + if(map){ + d['P'] = map.max + d['Q'] = map.min + } + } + + + } + }) + } + + + def findMaxAvgPrice(def city, def region, def alias){ + def set = null + try{ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210901 and region='${region}' and district_name in ('${aliasStr}') and root='anjuke.com'" + + set = new HashSet() + ElasticUtils.executeSqlQuery(sql,{data-> + set << data.avg_price + }) + }catch(Exception e){ + e.printStackTrace() + } + + if(set && set.size()>=3){ + def map = [:] + set = set.sort() + map.min = set[0] + map.max = set[set.size()-1] + return map + } + return null + } + + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findMinPrice(city){ + def avg_price = null + ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", { + avg_price = it.avg_price + }) + return avg_price + } + + def addMinMaxPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['V']){ + def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district") + if(dist){ + def alias = [] + if(!dist.alias){ + alias = [] + }else{ + alias = dist.alias + } + alias << dist.name + def map = findMaxAvgPrice(dist.city, dist.region, alias) + if(map){ + d['V'] = map.max + d['W'] = map.min + } + } + } + }) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index be10cf533..3d79e39fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,9 +19,44 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() - task.houseRun() +// task.houseRun() // task.statHouseUpdateCount() + + task.statDistHouse() + } + + def statDistHouse(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\大连有效小区确认_2256--查好-1.xlsx",{d-> + if(d['B']){ + def alias = [] + if(d['G']){ + def s = d['G'].replace("[", "").replace("]", "").replace("'","").replace('"',"") + def arrs = null + if(s){ + arrs = s.split(",") + } + if(arrs){ + arrs = arrs as List + if(!arrs.contains(d['D'])){ + arrs << d['D'] + } + alias = arrs + }else{ + alias << d['D'] + } + }else{ + alias << d['D'] + } + + def distCount = findHouseCount(d['B'], d['C'], alias, 20190101, 20221001) + if(distCount){ + d['E'] = distCount + } + println d['B']+" "+d['C']+" "+d['D']+" "+distCount + } + + }) } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy new file mode 100644 index 000000000..444fddbfc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +EsDataSource dataSource = new EsDataSource(); +String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + +findLogs() + +def findLogs(){ + def total = 0 + try{ + +// String sql = "select * from fangjia-log* where request like '/jnBank%'" +// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + def q = QueryBuilders.fuzzyQuery("request", "/jnBank") + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder); + println houses.size() + def set = new HashSet() + if(houses) { + for (def d : houses) { + def request = d.request + if(request){ + def arrs = request.split("&") + if(arrs){ + for(def a : arrs){ + if(a.contains("CITY_CD")){ + def list = a.split("=") + if(list && list.size()>=2){ + set.add(list[1]) + } + } + } + } + } + } + } + + println set + + +// ElasticUtils.executeSqlQuery(sql,{ data-> +// println data.request +// },"192.168.0.25") + }catch(Exception e){ + e.printStackTrace() + } + + return total +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 05d1cdc37..eac332041 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } @@ -223,6 +223,7 @@ class DistrictIndexBatch extends BaseBatch{ map.city = d.city map.region = d.region map.name = d.name + map.status = 0 map.avg_estate_charge = rMap.avg_estate_charge map.price_rank = rMap.price_rank diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index cbeaf0ef9..b63201d66 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -44,12 +44,20 @@ public class BeiKeCrawlByChrome { def run(){ - System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) - citys = citys.reverse() +// citys = citys.reverse() +// def index = 0 +// for(def c : citys){ +// index++ +// if(c.city=="合肥"){ +// break +// } +// } +// citys = citys[index..citys.size()-1] def start = 0 // for(def i=0;i=endDate.getTime()){ detailUrls << durl - parse(city, e) + parse(city, e, region) }else{ stop = true; break; @@ -176,13 +198,13 @@ public class BeiKeCrawlByChrome { } page++; }else{ - stop = true; + stop = true; } } return detailUrls } - def parse(city,e){ + def parse(city,e, region){ try { def title = e.select(".title a").text() if (title) { @@ -190,7 +212,7 @@ public class BeiKeCrawlByChrome { def arrs = title.split(" ") detail.city = city detail.district_name = arrs[0] - detail.region = findRegionByThirdPary(detail.city, detail.district_name) + detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } @@ -376,13 +398,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("合肥")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap - //cityMap.put(e.text(), "https:"+href) - } +// cityMap.put(e.text(), "https:"+href) +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy index cb64409e6..447a05bb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.math.NumberUtils +import java.util.Map.Entry; import java.util.regex.Pattern @@ -175,22 +177,23 @@ class ExportPaimai { def run(){ - def city = "宿迁" + def city = "合肥" def region = null def datas = findPaimaiData(city,region) println "总量:"+datas.size() def rows = [] def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层", "拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价", - "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"] + "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型", + "映射资产类型","映射拍卖状态","URL"] rows << header -// def set = new HashSet() + def set = new HashSet() def index = 0 for(def d : datas){ -// if(!set.contains(d.title)){ + if(!set.contains(d.title)){ index++ println index -// set << d.title + set << d.title def row = [] row << d.title.replace(",",",") row << d.province @@ -226,14 +229,19 @@ class ExportPaimai { row << d.lookCount row << d.callCount row << d.delayCount - row << d.url + row << d.rateDiscount//折价率 + row << d.mappingEstateType;//映射类型 + row << d.zcType;//拍卖类型:资产拍卖 司法拍卖 + row << d.mappingAgencyName//映射资产类型 + row << d.mappingStatus//映射拍卖状态 + row << d.url rows << row -// }else{ -// println d.title -// } + }else{ + println d.title +" "+d.url + } } - ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}).xlsx",rows,"数据") + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据") // batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK") println "生成完成........" @@ -452,18 +460,41 @@ class ExportPaimai { if(region){ pmquery.put("region",region) } +// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"])) // pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) - pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01"))) // Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE); - Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE); +// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE); // Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); - pmquery.put("url", new BasicDBObject('$regex', pattern)) +// pmquery.put("url", new BasicDBObject('$regex', pattern)) def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000) def rets = [] for(def db : dbcursor){ rets << db } + +// Collections.sort(rets, new Comparator() { +// @Override +// int compare(BasicDBObject o1, BasicDBObject o2) { +// if(o1.finishDate && o2.finishDate){ +// return -o1.finishDate.compareTo(o2.finishDate) +// }else{ +// return 0 +// } +// +// } +// }) + rets.sort {a,b-> + if(a.finishDate && b.finishDate){ + def v1 = a.finishDate + def v2 = b.finishDate + return v1.compareTo(v2) + }else{ + return 0 + } + } + rets = rets.reverse() return rets } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 86da3466b..c902929c4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -31,12 +31,13 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); task.run() // task.run1() - task.addDealRegion(); +// task.addDealRegion(); task.standardRegion() // task.updadteDealPriceField() } @@ -116,7 +117,7 @@ public class LianjiaCrawlByChrome { } def run(){ - System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) @@ -129,7 +130,7 @@ public class LianjiaCrawlByChrome { def standardRegion(){ def q = new BasicDBObject() - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) def dbcursor = DEAL_PRICE_COLL.find(q) for(def d : dbcursor){ @@ -152,8 +153,9 @@ public class LianjiaCrawlByChrome { def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") lastMonth = lastMonth +"-01" def q = new BasicDBObject() +// q.put("city", "福州") q.put("region", null) - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] @@ -187,8 +189,19 @@ public class LianjiaCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) cityMap.each{city, cityUrl-> //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, city, cityUrl) - println cityUrl +" 详情链接:"+detailUrls.size() + driver.get(cityUrl+"chengjiao/") + def doc = Jsoup.parse(driver.getPageSource()); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) + if(regions){ + for(def r : regions){ + def regionText = r.text() + def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") + + println "开始抓取【${city}】【${regionText}】【${href}】" + def detailUrls = getPageDeail(driver, city, href, regionText) + println cityUrl +" 详情链接:"+detailUrls.size() + } + } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -224,12 +237,12 @@ public class LianjiaCrawlByChrome { } - def getPageDeail(driver, city, cityUrl){ + def getPageDeail(driver, city, cityUrl, regionText){ def stop = false def page = 1 def detailUrls = [] while(!stop){ - def url = cityUrl + "chengjiao/pg${page}/" + def url = cityUrl + "pg${page}/" driver.get(url) def doc = Jsoup.parse(driver.getPageSource()); def elements = JQueryUtils.select(".listContent li", doc) @@ -238,10 +251,10 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e) + parse(city, e, regionText) }else{ stop = true; break; @@ -259,7 +272,7 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e){ + def parse(city,e, regionText){ try { def title = e.select(".title a").text() if (title) { @@ -267,7 +280,11 @@ public class LianjiaCrawlByChrome { def arrs = title.split(" ") detail.city = city detail.district_name = arrs[0] - detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(regionText){ + detail.region = regionText + }else{ + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + } if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } @@ -275,50 +292,61 @@ public class LianjiaCrawlByChrome { def roomStr = arrs[1] if (roomStr) { if (roomStr.indexOf("室") > 0) { - detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + def room = roomStr.substring(0, roomStr.indexOf("室")) + if(room && room.isNumber()){ + detail.room = room.toInteger() + } + } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hall && hall.isNumber()){ + detail.hall = hall.toInteger() + } } } - def areaStr = arrs[2].replace("平米", "") - if (areaStr) { - detail.area = areaStr.toDouble() - } - - def price = e.select(".totalPrice .number").text() - if (price && price.isNumber()) { - detail.total_price = price.toInteger() * 10000 - } - - if (detail.total_price && detail.area) { - detail.avg_price = (int) (detail.total_price / detail.area) - } - def tradeStr = e.select(".dealDate").text() - if (tradeStr) { - detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") - } - detail.source = "链家在线" - detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() + if("车位"!=roomStr){ + if(arrs && arrs.size()>=3){ + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() } } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + detail.url = e.select(".title a").attr("href") + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -446,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 7a314a59c..83ceb7171 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -8,8 +8,10 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.rabbitmq.client.AMQP +import com.rabbitmq.client.impl.AMQImpl import java.text.SimpleDateFormat import java.util.concurrent.CountDownLatch @@ -29,18 +31,20 @@ class PaimaiDataSend { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { def task = new PaimaiDataSend() -// task.jdRun() +// task.jdRunByMongo() + // task.aliZcRun() -// task.aliZcRun2() + task.aliZcRun2() // task.aliFpRun() // task.rmfysszcRun() +// task.jdRun() // task.checkUrl() - task.paimaiSendToEs() +// task.paimaiSendToEs() } @@ -64,7 +68,9 @@ class PaimaiDataSend { def rmfysszcRun(){ def index = 0 def q = new BasicDBObject() -// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml") +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) + q.put("province","江苏省") +// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -95,11 +101,22 @@ class PaimaiDataSend { } } - def jdRun(){ + def jdRunByMongo(){ + def pm_q = new BasicDBObject() + pm_q.put("city","宿迁") + pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) + pm_q.put("mappingStage", null) + def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) + println dbcursor.count() + for(def d : dbcursor){ + sendJd(d.url) + } + } + + def sendJd(url){ def index = 0 def q = new BasicDBObject() - q.put("city","宿迁市") -// q.put("url","https://paimai.jd.com/108245329") + q.put("url", url) def dbcursor = jd_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -110,6 +127,7 @@ class PaimaiDataSend { JSON result = new JSONObject() result.put("content", d.desc_content) + result.put("region", d.region) def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) result.put("bid_content", bid_content.toString()) @@ -131,20 +149,97 @@ class PaimaiDataSend { jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) } } + def jdRun(){ + def index = 0 + def q = new BasicDBObject() +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) + q.put("province","江苏") +// q.put("parser_status", new BasicDBObject('$ne',3)) +// q.put("url","https://paimai.jd.com/115669796") + def dbcursor = jd_coll.find(q).batchSize(3000) + def rets = [] + for(def d : dbcursor){ + index++ + println "jd:"+index + JSON json = new JSONObject() + json.put("project", "paimai_jd") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.desc_content) + result.put("region", d.region) + + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("bid_content", bid_content.toString()) + + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("content_json", d.content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content.content) + json.put("result", result) + + rets << json + if(rets.size()==10000){ + println "共:"+rets.size() + sendJD(rets) + rets.clear() + } + } + + if(rets.size()>0){ + println "共:"+rets.size() + sendJD(rets) + } + } + + def sendJD(def rets){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i0){ + println "共:"+rets.size() + send(rets) + } + } + + def send(def rets){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) for(def i=0;i0){ + println "共:"+rets.size() + send(rets) + } + +// println "共:"+rets.size() +// +// def totalCount = new AtomicLong() +// final CountDownLatch latch = new CountDownLatch(rets.size()) +// for(def i=0;i + def m = [:] + m.region = r.DISTNAME + m.cityId = r.CITYID + m.distId = r.DISTID + list << m + }) + return list; + } + + def run(){ def rets = findAllCity() for(def city : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy new file mode 100644 index 000000000..e27c6adce --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy @@ -0,0 +1,214 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class MingShengStat { + + def cityLevel0 = ["北京","上海","广州","深圳"] + def cityLevel1 = ["成都","重庆","杭州","武汉","西安","郑州","青岛","长沙","天津","苏州","南京","东莞","沈阳","合肥","佛山"] + def cityLevel2 = ["昆明","福州","无锡","厦门","哈尔滨","长春","南昌","济南","宁波","大连","贵阳","温州","石家庄","泉州","南宁","金华","常州","珠海","惠州","嘉兴","南通","中山","保定","兰州","台州","徐州","太原","绍兴","烟台","廊坊"] + def cityLevel3 = ["海口","汕头","潍坊","扬州","洛阳","乌鲁木齐","临沂","唐山","镇江","盐城","湖州","赣州","漳州","揭阳","江门","桂林","邯郸","泰州","济宁","呼和浩特","咸阳","芜湖","三亚","阜阳","淮安遵义","银川","衡阳","上饶","柳州","淄博","莆田","绵阳","湛江","商丘","宜昌", + "沧州","连云港","南阳","蚌埠","驻马店","滁州","邢台","潮州","秦皇岛","肇庆","荆州","周口","马鞍山","清远","宿州","威海","九江","新乡","信阳","襄阳","岳阳","安庆","菏泽","宜春","黄冈","泰安","宿迁","株洲","宁德","鞍山","南充","六安","大庆","舟山"] + def cityLevel4 = ["常德","渭南湖","孝感","丽水","运城","德州","张家口","鄂尔多斯","阳江","泸州","丹东","曲靖","乐山","许昌","湘潭","晋中","安阳","齐齐哈尔","北海","宝鸡","抚州","景德镇","延安","三明","抚顺亳州","日照","西宁","衢州","拉萨","淮北","焦作","平顶山","滨州","吉安","濮阳","眉山","池州","荆门","铜仁","长治", + "衡水","铜陵","承德","达州","邵阳","德阳","龙岩","南平","淮南","黄石","营口","东营","吉林","韶关","枣庄","包头","怀化","宣城","临汾","聊城","梅州","盘锦","锦州","榆林","玉林","十堰","汕尾","咸宁","宜宾","永州","益阳","黔南州","黔东南","恩施","红河","大理","大同","鄂州","忻州","吕梁","黄山","开封","郴州","茂名","漯河","葫芦岛","河源","娄底","延边"] + def cityLevel5 = ["汉中","辽阳","四平","内江","六盘水","安顺","新余","牡丹江","晋城","自贡","三门峡","赤峰","本溪","防城港","铁岭","随州","广安","广元","天水","遂宁","萍乡","西双版纳","绥化","鹤壁","湘西","松原","阜新","酒泉","张家界","黔西南","保山","昭通","河池","来宾","玉溪","梧州","鹰潭","钦州","云浮","佳木斯","克拉玛依","呼伦贝尔","贺州", + "通化","阳泉","朝阳","百色","毕节","贵港","丽江","安康","通辽","德宏","朔州","伊犁","文山","楚雄","嘉峪关","凉山","资阳","锡林郭勒盟","雅安","普洱","崇左","庆阳","巴音郭楞(巴州)","乌兰察布","白山","昌吉","白城","兴安盟","定西","喀什","白银","陇南","巴彦淖尔","巴中","鸡西","乌海","临沧","海东","张掖","商洛","黑河","哈密","吴忠","阿克苏"] + + def ajkNewDistsMap = ["北京": 538, "上海": 1928, "广州": 1834, "深圳": 1149, "武汉": 2181, "大连": 866, "南京": 1442, "杭州": 2245, "太原": 497, "石家庄": 723, "重庆": 2700, "西安": 1693, "福州": 305, "济南": 1003, "宁波": 1304, "成都": 3004, "天津": 1614, "昆明": 755, "南昌": 697, "青岛": 1843, "厦门": 253, "郑州": 1385, "长沙": 1811, "长春": 741, "合肥": 1265, "呼和浩特": 462, "沈阳": 456, "贵阳": 499, "拉萨": 82, "宝鸡": 249, "汕头": 411, "苏州": 1531, "温州": 566, "泉州": 565, "邯郸": 314, "唐山": 442, "沧州": 248, "洛阳": 560, "南阳": 422, "襄阳": 302, "吕梁": 79, "大同": 171, "潍坊": 739, "南通": 735, "衡阳": 425, "中山": 590, "宜昌": 360, "许昌": 384, "盐城": 459, "济宁": 510, "龙岩": 227, "上饶": 288, "马鞍山": 199, "赣州": 385, "金华": 479, "常德": 394, "延边": 71, "泰州": 304, "三亚": 456, "淮安": 488, "咸宁": 307, "无锡": 703, "常州": 470, "镇江": 382, "烟台": 917, "绍兴": 714, "莆田": 261, "东莞": 729, "佛山": 1161, "株洲": 285, "黄石": 151, "孝感": 379] + def bkNewDistsMap = ["北京": 316, "上海": 2002, "广州": 1350, "深圳": 1261, "武汉": 2308, "大连": 438, "南京": 579, "杭州": 1892, "太原": 418, "石家庄": 582, "重庆": 2122, "西安": 1608, "福州": 512, "济南": 779, "宁波": 592, "成都": 2849, "天津": 1039, "昆明": 1262, "南昌": 429, "青岛": 1275, "厦门": 387, "郑州": 1146, "长沙": 1434, "长春": 433, "合肥": 807, "呼和浩特": 294, "沈阳": 1137, "贵阳": 242, "拉萨": 3, "宝鸡": 50, "苏州": 1129, "温州": 126, "泉州": 410, "邯郸": 114, "唐山": 181, "沧州": 50, "洛阳": 192, "南阳": 290, "襄阳": 55, "大同": 62, "潍坊": 332, "南通": 509, "衡阳": 83, "中山": 516, "宜昌": 147, "许昌": 156, "盐城": 211, "济宁": 205, "龙岩": 5, "上饶": 160, "马鞍山": 120, "赣州": 236, "金华": 124, "常德": 116, "三亚": 164, "淮安": 278, "咸宁": 75, "无锡": 575, "常州": 221, "镇江": 310, "烟台": 548, "绍兴": 175, "莆田": 13, "东莞": 622, "佛山": 971, "株洲": 134, "黄石": 88, "孝感": 94] + def ljNewDistMap = ["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40] + def sfNewDistMap =["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40] + def chNewDistMap = ["北京": 408, "上海": 532, "广州": 559, "深圳": 763, "武汉": 775, "大连": 7, "南京": 1137, "杭州": 517, "太原": 293, "石家庄": 567, "重庆": 817, "西安": 439, "济南": 544, "宁波": 555, "成都": 1094, "天津": 754, "昆明": 446, "南昌": 379, "青岛": 840, "厦门": 267, "郑州": 960, "长沙": 767, "长春": 299, "合肥": 404, "呼和浩特": 136, "沈阳": 385, "贵阳": 330, "拉萨": 41, "宝鸡": 145, "汕头": 220, "苏州": 715, "温州": 422, "泉州": 312, "邯郸": 425, "唐山": 317, "沧州": 298, "洛阳": 268, "南阳": 233, "襄阳": 84, "吕梁": 38, "大同": 121, "潍坊": 421, "南通": 328, "衡阳": 329, "中山": 279, "宜昌": 158, "许昌": 161, "盐城": 165, "济宁": 251, "龙岩": 90, "上饶": 136, "马鞍山": 69, "赣州": 265, "金华": 252, "常德": 192, "延边": 59, "泰州": 122, "三亚": 120, "淮安": 150, "咸宁": 175, "无锡": 468, "常州": 259, "镇江": 223, "烟台": 420, "绍兴": 368, "莆田": 106, "东莞": 249, "佛山": 438, "株洲": 216, "黄石": 96, "孝感": 160] + def jwNewDistMap = ["北京": 510, "上海": 1353, "广州": 1193, "深圳": 697, "武汉": 1746, "大连": 539, "南京": 1219, "杭州": 2030, "太原": 631, "石家庄": 521, "重庆": 2165, "西安": 1518, "济南": 811, "宁波": 1023, "成都": 3192, "天津": 91, "昆明": 764, "南昌": 632, "青岛": 1420, "厦门": 274, "郑州": 1509, "长沙": 1474, "长春": 400, "合肥": 1112, "呼和浩特": 320, "沈阳": 648, "贵阳": 501, "拉萨": 75, "宝鸡": 217, "汕头": 350, "苏州": 1210, "温州": 628, "泉州": 407, "邯郸": 203, "唐山": 638, "沧州": 367, "洛阳": 509, "南阳": 311, "襄阳": 206, "吕梁": 48, "大同": 159, "潍坊": 1041, "南通": 883, "衡阳": 420, "中山": 457, "宜昌": 319, "许昌": 151, "盐城": 488, "济宁": 534, "龙岩": 161, "上饶": 205, "马鞍山": 181, "赣州": 380, "金华": 424, "常德": 305, "延边": 102, "泰州": 172, "三亚": 460, "淮安": 368, "咸宁": 223, "无锡": 509, "常州": 420, "镇江": 506, "烟台": 976, "绍兴": 608, "莆田": 226, "东莞": 1270, "佛山": 894, "株洲": 274, "黄石": 170, "孝感": 233] + def zgNewDistMap = ["南京": 711, "大连": 143, "青岛": 1117, "济南": 577, "昆明": 726, "中山": 466, "佛山": 852, "盐城": 384, "北京": 563, "无锡": 342, "东莞": 428, "烟台": 853, "厦门": 289, "南通": 325, "太原": 310, "泉州": 354, "贵阳": 1324, "马鞍山": 141, "上海": 1776, "苏州": 385, "郑州": 610, "常州": 230, "洛阳": 228, "潍坊": 795, "赣州": 238, "淮安": 268, "深圳": 1034, "长沙": 1252, "石家庄": 296, "金华": 156, "南昌": 383, "泰州": 255, "广州": 1193, "三亚": 341, "宁波": 475, "唐山": 382, "镇江": 495, "济宁": 330, "莆田": 213, "沧州": 155, "宝鸡": 173, "重庆": 634, "沈阳": 494, "长春": 182, "襄阳": 164, "天津": 556, "合肥": 366, "宜昌": 252, "温州": 428, "南阳": 379, "成都": 2452, "绍兴": 147, "呼和浩特": 337, "武汉": 845, "西安": 1407, "邯郸": 220, "常德": 259, "杭州": 568, "黄石": 143, "大同": 151, "许昌": 289, "延边": 72, "咸宁": 230, "拉萨": 60, "株洲": 225, "上饶": 210, "孝感": 229, "吕梁": 78, "汕头": 329, "龙岩": 150, "衡阳": 328] + + def fields = ["city","region","district_name","estate_type","area","avg_price","total_price","direction","fitment","room","hall","toilet","floor","height","build_structure","comm_id","dataType1","purpose","root","url", "p_date"] + def no_fields = ["build_date","build_type","floor_info"] + static main(args) { + def task = new MingShengStat(); +// task.run() +// task.addNewLevel +// task.addHouseRate() +// task.houseFieldStat() + task.addDistBuildStr() +// task.statFloorInfo() + } + + def addDistBuildStr(){ + def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"] + + def rows = [] + for(def c : citys){ + def row = [] + row << c + def cityInfoDto = CommonDatautils.findCityInfo(c) + if(cityInfoDto){ + def detailMap = cityInfoDto.district_detail + row << detailMap.finishBuildingDate.toDouble()*100 + } + rows << row + } + ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计-竣工时间.xlsx", rows, "字段统计") + } + + def houseFieldStat(){ + def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"] + def rows = [] + for(def c : citys){ + def row = [] + row << c + def total = findHouseCount(c) + for(def f : no_fields){ + def num = findValueByCity(c, f); + if(num){ + def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total) + row << rate + }else{ + row << 0 + } + } + rows << row + } + + ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计2.xlsx", rows, "字段统计") + } + + + + def statFloorInfo(){ + def file = "D:\\数据\\民生\\字段覆盖率统计-楼层.xlsx" + ExcelUtils.multiThreadingModify(file,{d-> + def total = findHouseCount(d['A']) + if(total>=100000){ + total = 100000 + } + def num = findHouseDatas(d['A']); + if(num){ + def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total) + d['C'] = rate + }else{ + d['C'] = 0 + } + }) + } + + def addHouseRate(){ + // def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\估值数据统计结果20220816.xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\民生\\城市统计.xlsx",{d-> + if(d['B']){ + def cityInfoDto = CommonDatautils.findCityInfo(d['B']) + d['G'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, cityInfoDto.price_coverage*100) + def dists = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city","region","name"]) + d['F'] = dists.size() + d['D'] = statNewDistCount(d['B']) + d['E'] = findHouseCount(d['B']) + } + }) + } + + def findValueByCity(city, field){ + def count = 0 + def sql = "select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell' and ${field} is not null" + ElasticUtils.executeSqlQuery(sql, { + count = it.count + println("$city\t${field} ${it.count}") + }) + return count + } + + + def findHouseDatas(city){ + def sql = "select floor_info from seawater where p_date >= 20200801 and city='${city}' and cat='sell' order by p_date desc limit 100000" + def f_count = 0 + def count = 0 + ElasticUtils.executeSqlQuery(sql, { + count++ + def s = it.floor_info + if(s && !s.contains("暂")){ + f_count = f_count +1 + } + println("${count} $city\t floor_info ${s}") + }) + return f_count + } + + def findHouseCount(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell'", { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def statNewDistCount(def city){ + def sum = 0 + if(ajkNewDistsMap.get(city)){ + sum = sum + ajkNewDistsMap.get(city) + } + if(bkNewDistsMap.get(city)){ + sum = sum + bkNewDistsMap.get(city) + } + if(ljNewDistMap.get(city)){ + sum = sum + ljNewDistMap.get(city) + } + if(sfNewDistMap.get(city)){ + sum = sum + sfNewDistMap.get(city) + } + if(chNewDistMap.get(city)){ + sum = sum + chNewDistMap.get(city) + } + if(jwNewDistMap.get(city)){ + sum = sum + jwNewDistMap.get(city) + } + if(zgNewDistMap.get(city)){ + sum = sum + zgNewDistMap.get(city) + } + return sum + } + + def addNewLevel(){ + ExcelUtils.modify("D:\\数据\\民生\\城市统计.xlsx",{d-> + if(d['B']){ + if(cityLevel0.contains(d['B'])){ + d['D'] = "一线" + }else if(cityLevel1.contains(d['B'])){ + d['D'] = "新一线" + }else if(cityLevel2.contains(d['B'])){ + d['D'] = "二线" + }else if(cityLevel3.contains(d['B'])){ + d['D'] = "三线" + }else if(cityLevel4.contains(d['B'])){ + d['D'] = "四线" + }else if(cityLevel5.contains(d['B'])){ + d['D'] = "五线" + } + } + }) + } + + def run(){ + def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"] + def rows = [] + for(def c : citys){ + def dbo = CommonDatautils.findAlCityDetail(c) + def row = [] + if(dbo){ + row << dbo.province + row << c + row << dbo.new_city_grade/1000 + }else{ + row << "" + row << c + row << "" + } + rows << row + } + ExcelUtils.write("D:\\数据\\民生\\城市统计.xlsx", rows, "数据") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy new file mode 100644 index 000000000..5312d274a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy @@ -0,0 +1,94 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.core.RegionData +import groovy.sql.Sql + +def newRegionCodeFile = "D:\\数据\\建行\\二期\\行政区处理_20220923.xlsx" + +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def specialCityMap = ["内蒙古##兴安":"兴安盟","四川##甘孜":"甘孜州","云南##文山":"文山州","青海##玉树":"玉树州"] + +//ExcelUtils.modify(newRegionCodeFile,{d-> +// if(d['A'] && !d['I']){ +// def city = specialCityMap.get(d["A"]+"##"+d["B"]) +// if(!city){ +// city = RegionData.formatCity(d['B']) +// } +// if(city){ +// def region = RegionData.formatRegion(city, d['E']) +// if(region){ +// d['I'] = city +// d['J'] = region +// } +// } +// } +//}) + +def newDatas = ExcelUtils.read(newRegionCodeFile,0,1) +def newDataMap = [:] +for(def d : newDatas){ + def key = d['I']+"##"+d['J'] + def rets = newDataMap.get(key) + if(!rets){ + newDataMap.put(key, d) + } +} + +def ccbMap = getCcb92City(sqlInstance) +ExcelUtils.modify("D:\\数据\\亲邻\\region_code.xlsx",{d-> + if(!d['C'] && d['D']){ + def dbo = CommonDatautils.findAlCityDetail(d['A']) + if(dbo){ + d['C'] = dbo.province + } + } +// if(!d['D']){ +// def m = newDataMap.get(d['A']+"##"+d['B']) +// if(m){ +// d['C'] = m['A'] +// d['D'] = m['B'] +// d['E'] = m['C'] +// d['F'] = m['D'] +// d['G'] = m['E'] +// } +// def m = ccbMap.get(d['A']+"##"+d['B']) +// if(m){ +// d['D'] = m.city +// d['E'] = m.cityId +// d['F'] = m.distId +// d['G'] = m.region +// } +// } +}) + +def getCcb92City(sqlInstance){ + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj" + def map = [:] + sqlInstance.eachRow(sql, { r -> + def city = getCityName(r.CITYID) + def region = RegionData.formatRegion(city, r.DISTNAME) + def m = [:] + m.city = city + m.region = r.DISTNAME + m.cityId = r.CITYID + m.distId = r.DISTID + map.put(city+"##"+region, m) + }) + return map; +} + +def getCityName(cityId){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def city = null + cityCodeDataMap.each{cityName, id-> + if(id == cityId){ + city = cityName + } + } + return city; +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 09db88159..e1976f1cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo @@ -15,10 +16,154 @@ import groovy.sql.Sql import org.bson.types.ObjectId import java.text.SimpleDateFormat +import java.util.regex.Pattern -def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164") -DBUtils.seaweed.save(dist) +DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018) + +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL) + +//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d-> +// def city = RegionData.formatCity(d['E']) +// if(city && !d['J']){ +// def region = RegionData.formatRegion(city, d['G']) +// d['I'] = city +// d['J'] = region +// } +//}) + + +//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d-> +// def city = RegionData.formatCity(d['D']) +// if(!city){ +// city = RegionData.formatCity(d['F']) +// } +// if(city && !d['H']){ +// def region = RegionData.formatRegion(city, d['B']) +// d['G'] = city +// d['H'] = region +// } +//}) + +//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] +//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx") + +//def fjwCitys = [] +//for(def d : datas){ +// fjwCitys << d['G'] +//} +//def rows = [] +//for(def d : datas){ +// if(citys.contains(d['G'])){ +// rows << d +// } +//} +//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域") + +//for(def c : citys){ +// if(!fjwCitys.contains(c)){ +// println c +// } +//} + +// +//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域") +// + +//def map = [:] +//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx") +//for(def row : datas1){ +// if(row['J']){ +// def codeMap = [:] +// codeMap.prov = row['B'] +// codeMap.city = row['F'] +// codeMap.region = row['H'] +// map.put(row['I']+"#"+row['J'], codeMap) +// } +//} +//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx") +//for(def row : datas2){ +// if(row['H']){ +// def codeMap = [:] +// codeMap.prov = row['E'] +// codeMap.city = row['C'] +// codeMap.region = row['A'] +// map.put(row['G']+"#"+row['H'], codeMap) +// } +//} + +//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> +// def key = d['B'] +"#"+d['C'] +// def codeMap = map.get(key) +// if(codeMap && !d['D']){ +// d['D'] = codeMap.prov +// d['E'] = codeMap.city +// d['F'] = codeMap.region +// } +//}) + +ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> + if(!d['D']){ + def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) + if(infos && infos.size()==1){ + def info = infos[0] + d['D'] = info.province_code + d['E'] = info.area_code + d['F'] = info.street_code + } + } +}) + + +def findTownshipInfo(city, region, coll){ + def q = new BasicDBObject() + q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) + q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) + + def dbcursor = coll.find(q) + def rows = [] + for(def d : dbcursor){ +// if(d.street_name.contains(region)){ + rows << d +// } + } + return rows +} + + + +//def cityMap = CommonDatautils.findAllCityProvince() +//def rows = [] +//rows << ["省", "市", "区域"] +//cityMap.each{c,p-> +// def regions = CommonDatautils.getCityRegions(c) +// for(def r : regions){ +// def row = [] +// row << p +// row << c +// row << r +// rows << row +// } +//} +//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域") + + + + +//def q = new BasicDBObject() +//q.put("root", "anjuke.com") +//def dbcursor = coll.find(q) +//def map = [:] +//for(def d : dbcursor){ +// map.put(d.city, d.city_short) +//} +//println map.size() +//println map + + +//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164") +//DBUtils.seaweed.save(dist) //DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist)) //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy new file mode 100644 index 000000000..e81f4cb67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy @@ -0,0 +1,142 @@ +package com.fangjia.script.gj.test + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018) + +DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018) + +DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018) + +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) +//Thread.sleep(60*60*1000) +//while (true){ + def q = new BasicDBObject() +// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"])) + q.put("status",new BasicDBObject('$ne', 1)) + def dbcursor = school_coll.find(q).batchSize(5000) + def rets = [] + for(def d : dbcursor){ + JSON json = new JSONObject() + json.put("project", d.project) + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def jsonStr = d.map_info + + jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1) + if(jsonStr.contains("\n")){ + jsonStr = jsonStr.replaceAll('\\n','') + } + result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr)) + + + result.put("type", d.type) + json.put("result", result) + + + rets << json + if(rets.size()==5000){ + println "共:"+rets.size() + send(rets, fixedThreadPool, school_coll) + rets.clear() + } + } + + if(rets.size()>0){ + println "共:"+rets.size() + send(rets, fixedThreadPool, school_coll) + rets.clear() + } +//} + + + +def send(def rets, def fixedThreadPool, def school_coll){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i0){ +// println "共:"+rets.size() +// send(rets, fixedThreadPool, school_coll) +// rets.clear() +//} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 926581423..24eeb3c76 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -647,9 +647,9 @@ class CZDataBatch extends BaseBatch{ } } - if(!priceMap){ - priceMap = findRegionRentPrice(priceMap, city, region, estate_type2) - } +// if(!priceMap){ +// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2) +// } //5室及以上价格小于2000去除 if(priceMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy new file mode 100644 index 000000000..a0c3ec24f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ElasticUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 补充学校经纬度 + * @author shenjian + * @since 2022/9/8 + */ +class FillLngLat { + + static ExecutorService executorService = Executors.newFixedThreadPool(4) + static void main(String[] args) { + ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + final def data = dataT + executorService.submit(new Runnable() { + @Override + void run() { + println(data) + def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + if (lngLat != null && lngLat.size() == 2 ) { + JSONObject jsonObject = new JSONObject() + jsonObject.put("lon", Double.valueOf(lngLat['lng'])) + jsonObject.put("lat", Double.valueOf(lngLat['lat'])) + EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) + } + } + }) + + }) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/ReGeoFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/ReGeoFromCommunity.groovy new file mode 100644 index 000000000..018bc06c0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/ReGeoFromCommunity.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.RegionData +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.poi.entity.Community +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import org.apache.commons.collections.CollectionUtils +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.stream.Collectors + +/** + * 逆解析第三方小区经纬度 + */ +class ReGeoFromCommunity { + + static final ExecutorService executorService = Executors.newFixedThreadPool(5) + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + Map map = new ConcurrentHashMap<>() + def rets = this.findCommunities() + CountDownLatch countDownLatch = new CountDownLatch(rets.size()) + final BulkRequest bulkRequest = new BulkRequest(); + for (def retTemp : rets) { + final def ret = retTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final String id = ret._id // 索引ID,并非小区ID + if (ret.location) { + String lon = ret.location.lon + String lat = ret.location.lat + def addressComponent = GaoDeUtils.reGeoLocation(lon + "," + lat) + if (addressComponent) { + def city_format = RegionData.formatCity(addressComponent.city) + def region_format = RegionData.formatRegion(city_format, addressComponent.district) + def township = addressComponent.township + Map obj = new HashMap<>() + obj.put("city", city_format) + obj.put("region", region_format) + obj.put("township", township) + + Map doc = new HashMap<>(); + doc.put("address_component", obj) + bulkRequest.add(new UpdateRequest("community", id).doc(doc)); + } + } + synchronized (map) { + map.put(id, id) + if (map.size() % 1000 == 0) { + atomicLong.addAndGet(map.size()) + EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT) + bulkRequest = new BulkRequest(); + map.clear() + println(atomicLong.intValue()) + } + } + } catch(Exception ex) { + + } finally { + countDownLatch.countDown() + } + } + }) + }; + countDownLatch.await(); + if (map.size() > 0) { + atomicLong.addAndGet(map.size()) + EsUtils.ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT) + bulkRequest = new BulkRequest(); + map.clear() + println(atomicLong.intValue()) + } + System.exit(1) + } + + static def findCommunities(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.existsQuery("address_component")) + .must(QueryBuilders.existsQuery("location")) + List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class, 200000) + return communities + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index f3c36ebd8..2bc9b09b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject @@ -27,15 +28,15 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.ShenJian) class SeaweedAroundPoi extends BaseBatch { - static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018) static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + //static ExecutorService executorService = Executors.newFixedThreadPool(2) static void main(String[] args) { def types = listAllType() //dealSpecialSeaweedAround(types) dealPoi(new BasicDBObject(), types) - executorService.shutdown() + //executorService.shutdown() } /** @@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch { } static dealPoi(DBObject query, def types) { - query.put("status", 1) - conn136.find(query).batchSize(5000).each {objTemp -> + query.put("canyin_status", 0) + conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp -> final def obj = objTemp - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + if (obj.lng) { + def lng = SeaweedUtils.formatLatLng(obj.lng) + def lat = SeaweedUtils.formatLatLng(obj.lat) + // {'lon': 121.810877, 'lat': 31.737642} +// def lng = 121.810877 +// def lat = 31.737642 + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) } - println("已处理小区:" + obj) } - }) - } +// }) +// } } static getNewSeaweed() { DBObject query = new BasicDBObject() query.put("cat", "district") query.put("visible", 0) + //query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"])) + //query.put("city", "连云港") //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) DBObject filter = new BasicDBObject() @@ -163,17 +171,18 @@ class SeaweedAroundPoi extends BaseBatch { public static final Map KEYWORDS_MAP = new HashMap() { { - put("餐饮", "05"); +// put("餐饮", "05"); // put("地铁", "150500"); // put("公交", "1507"); // put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); // put("写字楼", "120201"); // put("学校", "141200|141201|141202|141203|141204"); // put("银行", "1601|1603"); -// put("大型超市", "0604"); + put("大型超市", "0604"); // put("商场", "0601"); // put("餐饮", "0501|0502|0503|0504"); // put("旅游景点", "110000"); + put("公园广场", "110100|110101|110102|110103|110104|110105") // put("图书馆", "140500"); // put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); // put("丧葬设施", "071900|071901|071902|071903") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy new file mode 100644 index 000000000..66abaacf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiFromExcel { + + static void main(String[] args) { + def types = listAllType() + dealPoi(new BasicDBObject(), types) + } + + static dealPoi(DBObject query, def types) { + com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item -> + def city = item['H'] + def lng = item['AQ'] + def lat = item['AR'] + if (lng) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr) + } + Thread.sleep(1000) + print(lng + " " + lat) + } + }); + }; + + + + public static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + }; + + public static final Map KEYWORDS_MAP = new HashMap() { + { +// put("餐饮", "05"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校", "141200|141201|141202|141203|141204"); +// put("银行", "1601|1603"); + put("大型超市", "0604"); +// put("商场", "0601"); +// put("餐饮", "0501|0502|0503|0504"); +// put("旅游景点", "110000"); + put("公园广场", "110100|110101|110102|110103|110104|110105") +// put("图书馆", "140500"); +// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); +// put("丧葬设施", "071900|071901|071902|071903") + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy index ba2bfaaea..dd1a07103 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy @@ -1,6 +1,6 @@ -package com.fangjia.script.temp.shenjian.poi.entity; +package com.fangjia.script.temp.shenjian.poi.entity -public class Community extends MyId { +class Community extends MyLocation { private String id; private String city; private String region; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyLocation.groovy new file mode 100644 index 000000000..7fc56e0fe --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyLocation.groovy @@ -0,0 +1,16 @@ +package com.fangjia.script.temp.shenjian.poi.entity + +import org.elasticsearch.common.geo.GeoPoint + +class MyLocation extends MyId { + + GeoPoint location; + + GeoPoint getLocation() { + return location + } + + void setLocation(GeoPoint location) { + this.location = location + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy index b158c9351..1ac5a78fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy @@ -3,12 +3,14 @@ package com.fangjia.script.temp.shenjian.util import com.alibaba.fastjson.JSONObject import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils import com.fangjia.script.temp.shenjian.poi.entity.MyId +import com.fangjia.script.temp.shenjian.poi.entity.MyLocation import org.elasticsearch.action.DocWriteRequest import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.search.* import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.geo.GeoPoint import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll @@ -23,7 +25,7 @@ import org.slf4j.LoggerFactory */ class EsUtils { - private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() + public static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() private static final Logger logger = LoggerFactory.getLogger(EsUtils.class) /** @@ -35,7 +37,7 @@ class EsUtils { * @param * @return */ - public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { + public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { List result = new ArrayList<>(); // 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大 Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L)); @@ -43,7 +45,11 @@ class EsUtils { searchSourceBuilder.query(boolQuery); // TEST 用!!!!!!!!!!!!!!!!!!!!!! //searchSourceBuilder.size(1); - searchSourceBuilder.size(10000); + if (limitSize < 10000) { + searchSourceBuilder.size(limitSize) + } else { + searchSourceBuilder.size(10000) + } SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder); SearchResponse searchResponse = null; try { @@ -57,6 +63,13 @@ class EsUtils { for (SearchHit searchHit : searchHits) { T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); t.set_id(searchHit.getId()) + Map location = searchHit.getSourceAsMap().get("location") + if (location != null) { + Double lon = location["lon"] + Double lat = location["lat"] + GeoPoint geoPoint = new GeoPoint(lat, lon) + t.setLocation(geoPoint) + } result.add(t); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 2a2972c21..17b02bb6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -146,11 +146,15 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - params.key = this.getGaodeKey("v5/place/around") - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + params.key = this.getGaodeKey("v3/place/around") + //params.key = "25b2d459fd133e22f8abcebeac972692" + params.offset = "100" + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { - params.key = this.getGaodeKey("v5/place/around") - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + params.key = this.getGaodeKey("v3/place/around") + //params.key = "25b2d459fd133e22f8abcebeac972692" + params.offset = "100" + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() @@ -258,7 +262,7 @@ class GaoDeUtils { } if (json && json.info && json.info == "OK") { try { - List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class) return ret } catch(Exception ex) { ex.printStackTrace() @@ -334,10 +338,6 @@ class GaoDeUtils { /** * 经纬度转地址
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 - * @author yinjihuan - * @date 2016年9月9日 - * @param location - * @return */ static def getGaoDeAddress(location) { Map params = new HashMap() @@ -369,6 +369,41 @@ class GaoDeUtils { return null } + /** + * 地址逆解析 + * @param location + * @return 经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 + */ + static def reGeoLocation(location) { + Map params = new HashMap() + params.put("location", location) + String content + try { + params.put("key", this.getGaodeKey("v3/geocode/regeo")) + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } catch (Exception e) { + params.put("key", this.getGaodeKey("v3/geocode/regeo")) + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) + Map infoMap = DataHandleUtils.JsonToMap(json); + if (infoMap && infoMap.info == "OK") { + def regeocode = infoMap.regeocode + if (regeocode) { + return regeocode.addressComponent + } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return null + } + /** * 获取地址经纬度 */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 9a9fd90ad..91d3dee7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils - +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{ } def rowIndex = 0; + + DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018) + DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018) static main(args) { runBatch(new DataStandardBatch(), args) } @@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> +// if(city == "北京"){ println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) +// } } while(flag){ @@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{ if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ district = null; }else{ - district = getMatchNameFromMap(data.city, data.region, data.name, data.address) + district = getMatchNameFromMap(data) if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区” - district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address) + def copyObj = data + copyObj.name = data.name+"小区" + district = getMatchNameFromMap(copyObj) } - if(!district){ def newName = filterSpacialChar(data.name);//包含特殊符号 if(newName) { - district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address) + def copyObj = data + copyObj.name = newName.replaceAll("_", "") + district = getMatchNameFromMap(copyObj) if(!district){ def s1 = newName.substring(0, newName.indexOf("_")) if (s1) { //特殊符号前的内容匹配 //district = getMatchName(data.city, data.region, s1, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s1, data.address) + copyObj.name = s1 + district = getMatchNameFromMap(copyObj) if (!district) { - def s2 = newName.replace(s1, "").replaceAll("_", "") def newName2 = null; if (s2 && s2.length() >= 3) { //特殊符号后的内容匹配 // district = getMatchName(data.city, data.region, s2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s2, data.address) + copyObj.name = s2 + district = getMatchNameFromMap(copyObj) if (district) { type = 3; println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]" } } else { newName2 = newName.replaceAll("_", "") + copyObj.name = newName2 //district = getMatchName(data.city, data.region, newName2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, newName2, data.address) + district = getMatchNameFromMap(copyObj) if (district) { type = 4; println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]" @@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{ } if (!district) { - - district = getMatchNameFromMap(data.city, null, newName2, data.address) + copyObj.name = newName2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if(!district){ - district = getMatchNameFromMap(data.city, null, s1, data.address) + copyObj.name = s1 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 5; } else { if (s2 && s2.length() >= 3) { - district = getMatchNameFromMap(data.city, null, s2, data.address) + copyObj.name = s2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 6; } @@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{ }catch (Exception e){ addr = data.address; } - district = getMatchAddressFromMap(data.city, data.region, addr, data.name) + def copyObj = data + copyObj.address = addr + district = getMatchAddressFromMap(copyObj) if(district){ type = 8; }else{ - district = getMatchAddressFromMap(data.city, null, addr, data.name) + copyObj.region = null + district = getMatchAddressFromMap(copyObj) if(district){ type = 9; } @@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{ if(!district){ def districts = getMatchPingYing(data.city, data.region, data.name, addr); - district = checkMatchAddress(districts, addr); + district = checkMatchAddress(districts, addr, data); if(district){ type = 10; }else if(districts.size()==1){ - district = districts[0]; - type = 14 + district = getFilerMatchDistrict(districts[0], data); + if(district){ + type = 14 + } } } if(!district){ - district = getOpenApiMatch(data.city, data.region, data.name, addr); + def copyObj = data + copyObj.address = addr + district = getOpenApiMatch(copyObj) if(!district && addr){ - district = getOpenApiMatch(data.city, data.region, addr, data.name); + district = getOpenApiMatch(copyObj) } if(district){ if(data.region != district.region){ @@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{ return null; } - def checkMatchAddress(def districts, def address){ + def checkMatchAddress(def districts, def address, def data){ def district = null; if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){ def tempDist = districts[0]; @@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(district){ + return getFilerMatchDistrict(district, data) + } return district; } @@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, String address){ - //查询房价网是否存在该小区 - def dbo = findSameNameFromMap(city, region, disName); + def getMatchNameFromMap(def data){ + //查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price + def dbo = findSameNameFromMap(data.city, data.region, data.name); def addrFlag = null - if(address){ - addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(data.address){ + addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(addrFlag){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } if(!dbo){ //别名 - def distrs = findAliasNameFromMap(city, region, disName); + def distrs = findAliasNameFromMap(data.city, data.region, data.name); if(distrs && distrs.size()==1){ dbo = distrs[0] - }else if(distrs && distrs.size()>1 && address){ + }else if(distrs && distrs.size()>1 && data.address){ for(def d : distrs){ - if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ + if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } //地址 if(!dbo){ - def result = findAddressFromMap(city, region, disName) + def result = findAddressFromMap(data.city, data.region, data.name) if(result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && address){ + }else if(result && result.size()>1 && data.address){ for(def d : result){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } } + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } return dbo; } + def getFilerMatchDistrict(dbo, data){ + def types = [] + if(data.estate_type){ + types = getEstateTypes(data.estate_type) + } + if(types.size()>0 && types.size() == 1){ + if(types.contains(dbo.estate_type)){ + return dbo; + }else{ + def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","物业类型不匹配") + THIRD_MATCH_FILTER_COLL.save(data) + } + return null + } + }else{ + def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger()) + def price = priceMap.get(dbo.estate_type) + if(price && data.avg_price && data.avg_price.toString().isNumber()){ + def pre = (data.avg_price-price)*100/price + if(pre>=-20 && pre<=20){ + return dbo; + }else{ + def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","房源价格过滤") + THIRD_PRICE_FILTER_COLL.save(data) + } + return null + } + } + } + return dbo + } + + + def findHousePrice(def city, def region, def name, def startDate){ + def typePriceMap = [:] + try{ + String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'" + def houseMap = [:] + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.estate_type){ + def key = "" + if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){ + key = "普通住宅" + }else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){ + key = "商住" + }else if( data.estate_type.contains("别墅")){ + key = "别墅" + }else{ + key = "其它" + } + def houses = null + if(key){ + houses = houseMap.get(key) + if(!houses){ + houses = [] + } + } + if(data.avg_price){ + houses << data.avg_price + houseMap.put(key, houses) + } + } + }) + if(houseMap){ + houseMap.each{k,v-> + if(v && v.size()>=3){ + def price = (int)(v.sum()/v.size()) + typePriceMap.put(k, price) + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return typePriceMap + } + + def getEstateTypes(str){ + def types = [] + if(str){ + if(str.contains("住宅") || str.contains("公寓")){ + types << "普通住宅" + } + if(str.contains("商住") || str.contains("商务")){ + types << "商住" + } + if(str.contains("别墅")){ + types << "别墅" + } + } + return types; + } /*def getMatchAddress(def city, def region, def address, def name, def cat){ def dbo = CommonDatautils.findSeaweed(city, region, address, cat); @@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{ } return dbo; }*/ - def getMatchAddressFromMap(def city, def region, def address, def name){ + def getMatchAddressFromMap(def copyObj){ def dbo = null - if(address && isStandardRoad(address)){ - dbo = findSameNameFromMap(city, region, address); + def name = copyObj.name + if(copyObj.address && isStandardRoad(copyObj.address)){ + dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address); } if(!dbo){ def result = null - if(address && isStandardRoad(address)){ - result = findAddressFromMap(city, region, address); + if(copyObj.address && isStandardRoad(copyObj.address)){ + result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address); } if(result && result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && name){ + }else if(result && result.size()>1 && copyObj.name){ def score = null; for(def d : result){ - if(d.name == name){ + if(d.name == copyObj.name){ dbo = d; break; }else{ @@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{ } } if(!dbo){ - result = findAddrAliasFromMap(city, region, address); + result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ @@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(dbo){ + return getFilerMatchDistrict(dbo, copyObj) + } + } return dbo; } @@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{ if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] } } } @@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{ } - def getOpenApiMatch(def city, def region, def name, def address){ - if(StringUtils.isNotEmpty(name)){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false) + def getOpenApiMatch(def data){ + if(StringUtils.isNotEmpty(data.name)){ + def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) if(jsonObject && jsonObject.credit>0 ){ - return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit] + def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } + return dbo } } return null; @@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) +// qdata.put("city","北京") +// qdata.put("source","安居客") +// qdata.put("name",Pattern.compile("金隅澜湾.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -948,9 +1088,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } @@ -962,9 +1102,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -990,18 +1130,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1016,18 +1156,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1052,9 +1192,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1068,9 +1208,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1093,9 +1233,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1106,9 +1246,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 6ff3f087f..e76898c7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "城市房产"// - def citys = findCrawlCity(source, 202205); + def source = "安居客"// + def citys = findCrawlCity(source, 202208); for(def city : citys){ - def count1 = findCount(source, city, 202205); - def count2 = findCount(source, city, 202206); + def count1 = findCount(source, city, 202208); + def count2 = findCount(source, city, 202209); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 91e44595b..f954743e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils @@ -11,31 +14,38 @@ import org.bson.types.ObjectId * 福州农商行 * 将抓取的小区均价同步到第三方均价表中 */ -class SynchDistrictPrice { +@BatchOwner(Watcher.GaoJuan) +class SynchDistrictPrice extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - //"fang.com":"搜房", // def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] - //"fang.com":"搜房", def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - //,"城市房产":"城市房产" - def esSources = ["365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","吉屋":"吉屋"]// + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" + def esSources = ["中原地产":"中原地产"] + + static main(args) { - def task = new SynchDistrictPrice() -// task.run() - task.run1() + runBatch(new SynchDistrictPrice(), args); + } + + @Override + public void process(Object args) { + run1() } def run1(){ + + println "starting......" + def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def citys = CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ - def rets = findAllEsData(city, esSource, mgSource) + def rets = findAllEsData(city, esSource, mgSource, mdate) println city +" "+rets.size() synchData(city, rets, mgSource) } @@ -43,11 +53,11 @@ class SynchDistrictPrice { } - def findAllEsData(city,esSource, mgSource){ + def findAllEsData(city,esSource, mgSource, mdate){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d-> - def s = d.city +"#"+d.region+"#"+d.name + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type if(!names.contains(s)){ names << s def data = new BasicDBObject() @@ -57,6 +67,7 @@ class SynchDistrictPrice { data.put("address", d.address) data.put("avg_price", d.avg_price) data.put("source", mgSource) + data.put("estate_type",d.property_type) data.put("s_date", SHORT_DATE) data.put("url", d.url) data.put("c_date", new Date()) @@ -134,7 +145,7 @@ class SynchDistrictPrice { println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source) if(obj){ updateIndex++ // thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) @@ -142,6 +153,7 @@ class SynchDistrictPrice { def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) + udata.put("estate_type", r.estate_type) udata.put("m_date",new Date()) thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) @@ -171,13 +183,16 @@ class SynchDistrictPrice { } - def findIsExists(def city, def region, def name, def sDate, def source){ + def findIsExists(def city, def region, def name, estate_type, def sDate, def source){ def q = new BasicDBObject() q.put("source", source) q.put("s_date", sDate) q.put("city", city) q.put("region", region) q.put("name", name) + if(estate_type){ + q.put("estate_type",estate_type) + } def obj = thirdPartyColl.findOne(q) return obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index a5baaf550..99355ab2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202206); - def count2 = findTotalCount(source, 202207); + def count1 = findTotalCount(source, 202208); + def count2 = findTotalCount(source, 202209); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 89e0e9483..3efe71b67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ void process(Object args) { println SHORT_DATE def citys = CommonDatautils.findAllCity(); + def weekend = DateUtils.getFirstDayOfWeek(new Date()); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); def cityIndex = 0; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 3d4e855f2..68cd266c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -354,12 +354,13 @@ public class OpenApiClient { params.put("city", city); params.put("address", address); params.put("token", TOKEN); - params.put("__refresh_cache","1") +// params.put("__refresh_cache","1") - String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + //192.168.0.100:30115 + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8") println("${city} ${address} 程序调用异常") Thread.sleep(6000) } @@ -369,7 +370,7 @@ public class OpenApiClient { println("${city} ${address}") } } catch (Exception e) { - println("${city} ${address} ${e.getMessage()}") + println("[http error] ${city} ${address} ${e.getMessage()}") e.printStackTrace(); } return null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 2f146d11f..fdcf818c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + def static GAO_DE_AK = "90010d53a16ce9df544acab297645106" /** * 经纬度转地址
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy index 58655f17f..5c42ae8fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy @@ -38,8 +38,8 @@ class BasicInfoDataSynch extends BaseBatch{ synchRegion() synchTown() synchDict() - synchDistrict("district") - synchDistrict("office") +// synchDistrict("district") +// synchDistrict("office") } def synchCitys(){ @@ -299,7 +299,7 @@ class BasicInfoDataSynch extends BaseBatch{ } def synchDistrictByCity(fjCity, districts, cat){ - def zsDistrictCursor = ZS_SEAWEED_CONN.find(new BasicDBObject("city", fjCity).append("cat", cat)) + def zsDistrictCursor = ZS_SEAWEED_CONN.find(new BasicDBObject("city", fjCity).append("category", cat)) def zsDistricts = [] def zsDistrictMap = [:] def fjDistrictMap = [:] @@ -322,7 +322,6 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("estate_type",fjDistrict.estate_type2) udata.put("township",fjDistrict.township) udata.put("alias",fjDistrict.alias) - udata.put("cat",cat) udata.put("update_date",new Date().getTime()) if(fjDistrict.addr_alias){ udata.put("addr_alias",fjDistrict.addr_alias) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java index 8192b359c..14922257d 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -50,6 +50,21 @@ public class AmapPoiDetail { private BizExtDTO bizExt; @com.alibaba.fastjson.annotation.JSONField(name = "photos") private List photos; + @com.alibaba.fastjson.annotation.JSONField(name = "business") + private BusinessDTO business; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BusinessDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "business_area") + private String businessArea; + @com.alibaba.fastjson.annotation.JSONField(name = "keytag") + private String keytag; + @com.alibaba.fastjson.annotation.JSONField(name = "opentime_today") + private String opentimeToday; + @com.alibaba.fastjson.annotation.JSONField(name = "opentime_week") + private String opentimeWeek; + } @lombok.NoArgsConstructor @lombok.Data @@ -64,7 +79,7 @@ public class AmapPoiDetail { @lombok.Data public static class PhotosDTO { @com.alibaba.fastjson.annotation.JSONField(name = "title") - private List title; + private String title; @com.alibaba.fastjson.annotation.JSONField(name = "url") private String url; } diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/paimai9.png b/projects/fangjia-static/src/main/webapp/images/fangjia/paimai9.png new file mode 100644 index 000000000..18b5ff7c9 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/paimai9.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png index 366fdf4f2..654b2bce2 100644 Binary files a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai1.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai1.png new file mode 100644 index 000000000..3ce97e320 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai1.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai10.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai10.png new file mode 100644 index 000000000..972965007 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai10.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai9.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai9.png new file mode 100644 index 000000000..3aa0f166d Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai9.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_01.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_01.png new file mode 100644 index 000000000..2408d4d28 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_01.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_2.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_2.png new file mode 100644 index 000000000..e3efa58c6 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_2.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_3.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_3.png new file mode 100644 index 000000000..ff6df91a3 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_3.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_5.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_5.png new file mode 100644 index 000000000..dfc721a68 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_5.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_6.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_6.png new file mode 100644 index 000000000..a66a7c596 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_6.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_7.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_7.png new file mode 100644 index 000000000..dfcaa8164 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/paimai_7.png differ diff --git a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/main.css b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/main.css index cbef6ea34..d03baaaed 100644 --- a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/main.css +++ b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/main.css @@ -80,11 +80,15 @@ .adBox p small{line-height:26px;} .adBox .btnGroup{overflow:hidden;width:230px;margin:0 auto;} + .adBox01{width:1200px;height:202px;background:url(../../../images/fangjia/v5/adBox01_bg.png) no-repeat;margin-bottom:0;} .adBox01 p label{color:#fff;} .adBox01 p small{color:#fff;} .adBox01 .link01_btn{margin:0 auto;margin-top:20px;} + + + .adBox02{width:100%;height:90px;background:url(../../../images/fangjia/v5/adBox02_bg.png) no-repeat;margin-bottom:0;} .adBox02 > div{width:1200px;height:100px;margin:0 auto;} .adBox02 > div > div{float:left;} @@ -167,6 +171,7 @@ .firstOne .column .column_center img{margin:0 auto;} .firstOne .column .column_center > p{text-align:center;line-height:32px;font-size:18px;width:1000px;margin:40px auto;} +.firstOne .column .column_center2 > p{line-height:32px;font-size:18px;width:1000px;} .firstOne .column .column_center > div{overflow:hidden;width:795px;margin:0 auto;} .firstOne .column .column_center > div > img{float:left;margin:0 40px;} .firstOne .column .column_center > div > div{float:left;} diff --git a/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java b/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java index 1aacf834b..3238d95f8 100644 --- a/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java +++ b/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java @@ -309,7 +309,7 @@ public class HouseToCController { params.put("token", token); params.put("size", query.getArea()); params.put("app", "youda");// open淇濆瓨浼颁环璁板綍 - params.put("youDa", true); + //params.put("youDa", true); if (houseInfo.getFloor() != 0 && houseInfo.getHeight() != 0) { params.put("floor", houseInfo.getFloor()); params.put("maxFloor", houseInfo.getHeight()); @@ -601,7 +601,7 @@ public class HouseToCController { params.put("name", name); params.put("type", type * (-1)); params.put("evalDate", DateUtils.date2Str(houseInfo.getEvalTime())); - params.put("youDa", true); + //params.put("youDa", true); DistrictEvaluateSingleDto result = openApiProxyService.getProxyService(FshApiRemoteService.class) .getFshDistPriceTrends(params);