diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java index 2c8899d95..3d3ca01eb 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/District2Controller.java @@ -693,12 +693,6 @@ public class District2Controller { if (null != district.getLat2() && district.getLat2() > 0) { data.put("lat2", district.getLat2()); } - if (null != district.getLng3() && district.getLng3() > 0) { - data.put("lng3", district.getLng3()); - } - if (null != district.getLat3() && district.getLat3() > 0) { - data.put("lat3", district.getLat3()); - } if(district.getLat2()>0 && district.getLng2() >0){ Map map = GeocodeUtils.geoconvBaiduToGoogle(district.getLng2()/10000000000d, district.getLat2()/10000000000d); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java index dd04da3dd..659fe43c9 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java @@ -2034,14 +2034,6 @@ public class DistrictController { if (null != district.getLat2() && district.getLat2() > 0) { data.put("lat2", district.getLat2()); } - // 新加腾讯地图 - if (null != district.getLng3() && district.getLng3() > 0) { - data.put("lng3", district.getLng3()); - } - if (null != district.getLat3() && district.getLat3() > 0) { - data.put("lat3", district.getLat3()); - } - if (district.getLat2() > 0 && district.getLng2() > 0) { Map map = GeocodeUtils.geoconvBaiduToGoogle(district.getLng2() / 10000000000d, @@ -2054,12 +2046,8 @@ public class DistrictController { district.setLat(lat); district.setLng(lng); } - district.setLat3(lat); - district.setLng3(lng); data.put("lat", district.getLat()); data.put("lng", district.getLng()); - data.put("lat3", district.getLat3()); - data.put("lng3", district.getLng3()); } data.put("name", district.getName()); @@ -3305,8 +3293,6 @@ public class DistrictController { put("lng2", "百度经度"); put("lat", "高德纬度"); put("lng", "高德经度"); - put("lat3", "腾讯纬度"); - put("lng3", "腾讯经度"); put("loopLineInfo", "环线位置"); put("intelligenceLevel", "智能水平"); } diff --git a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl index 62474bcc9..52454f04e 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl @@ -192,31 +192,6 @@ 娉細濉叆鏁存暟锛屽393422150000 - - - <#if markMap?? && markMap["lng3"]?? && markMap["lng3"]?is_number && markMap["lng3"] == 1> - - <#else> - - - 缁忓害(鑵捐)* - - - - - - - - <#if markMap?? && markMap["la3t"]?? && markMap["lat3"]?is_number && markMap["lat3"] == 1> - - <#else> - - - 绾害(鑵捐)* - - 娉細濉叆鏁存暟锛屽393422150000 - - <#if markMap?? && markMap["address"]?? && markMap["address"]?is_number && markMap["address"] == 1> diff --git a/projects/fangjia-bkoff/src/main/webapp/district/list.ftl b/projects/fangjia-bkoff/src/main/webapp/district/list.ftl index 60e42be0a..93466d73a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/list.ftl @@ -202,7 +202,6 @@ - @@ -671,10 +670,7 @@ <#if district.lat??>${district.lng/10000000000}锛�${district.lat/10000000000} - - - <#if district.lat3??>${district.lng3/10000000000}锛�${district.lat3/10000000000} - + diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl index f250af3dc..207cd11ae 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/addOffice.ftl @@ -89,23 +89,6 @@ 娉細濉叆鏁存暟锛屽393422150000 - - - - - 缁忓害(鑵捐)* - - - - - - - - 绾害(鑵捐)* - - 娉細濉叆鏁存暟锛屽393422150000 - - 鍦板潃* diff --git a/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl b/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl index e62a5cc71..8dd9ca8ee 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district2/editOffice.ftl @@ -145,33 +145,6 @@ 娉細濉叆鏁存暟锛屽393422150000 - - - - - <#if markMap?? && markMap["lng3"]?? && markMap["lng3"] == 1> - - <#else> - - - 缁忓害(鑵捐)* - - - - - - - <#if markMap?? && markMap["lat3"]?? && markMap["lat3"] == 1> - - <#else> - - - 绾害(鑵捐)* - - 娉細濉叆鏁存暟锛屽393422150000 - - - <#if markMap?? && markMap["address"]?? && markMap["address"] == 1> diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index adfb2c7c8..4aaabb88f 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -25,8 +25,6 @@ import java.util.zip.GZIPOutputStream; import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.dictionary.py.Pinyin; -import com.sun.javafx.scene.transform.TransformUtils; -import net.sf.json.JSON; import net.sf.json.JSONArray; import net.sf.json.JSONNull; import net.sf.json.JSONObject; diff --git a/projects/fangjia-coresite/src/main/webapp/cpdb/index.ftl b/projects/fangjia-coresite/src/main/webapp/cpdb/index.ftl index 406f23e1f..74b7c1da7 100644 --- a/projects/fangjia-coresite/src/main/webapp/cpdb/index.ftl +++ b/projects/fangjia-coresite/src/main/webapp/cpdb/index.ftl @@ -142,7 +142,6 @@

CPDB澶ф暟鎹簱锛屽寘鍚�70澶氫釜缁村害鐨勬暟鎹紝鍚勭琛屼笟鍙婂満鏅殑搴旂敤

-
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 32e526e32..e7c2195ef 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -65,8 +65,6 @@ public class District implements SeaweedEntity { private Long lng; private Long lat2;//百度 private Long lng2; - private Long lat3;//腾讯 - private Long lng3; private String developer; private String estateCompany; private String allArea; @@ -1075,21 +1073,6 @@ public class District implements SeaweedEntity { this.lng2 = lng2; } - public void setLat3(Long lat3) { - this.lat3 = lat3; - } - @Column(name = "lat3") - public Long getLat3() { - return lat3; - } - public void setLng3(Long lng3) { - this.lng3 = lng3; - } - @Column(name = "lng3") - public Long getLng3() { - return lng3; - } - @Column(name = "finish_building_date") public String getFinishBuildingDate() { return finishBuildingDate; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java index 40a303f11..d5d69a1af 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/FshController.java @@ -386,24 +386,24 @@ public class FshController { Double oneMonthMarkup = null; Evaluation evaluation = null; if(model.isYouDa()){ - if("上海".equals(model.getCity())){ - YdDealPrice ydDealPrice = ydDealPriceService.findYdDealPrice(model.getCity(), model.getDistrict(), model.getName()); - if(ydDealPrice == null){ - mav.setModel(BasicModel.buildOKModel().setMsg("该小区暂不支持估价").markNoData()); - return mav; - }else{ - evaluation = evaluationService.ydEvaluate(district, buildYdEvaluation(model, request), model.getCalDate(),ydDealPrice.getPrice()); - //近30天的涨跌幅:取最近1个月至3个月之间最近的那期价格和当前均价相比 - Date end = DateUtil.getDateOfLastMonth(new Date(),-1); - Date start = DateUtil.getDateOfLastMonth(new Date(),-3); - List ydDealPrices = ydDealPriceService.findYdDealPrices(model.getCity(), model.getDistrict(), model.getName(),start,end); - if(ydDealPrices!=null && ydDealPrices.size()>0){ - oneMonthMarkup = (double)(ydDealPrices.get(0).getPrice()-ydDealPrice.getPrice())/ydDealPrice.getPrice(); - } - } - }else{ +// if("上海".equals(model.getCity())){ +// YdDealPrice ydDealPrice = ydDealPriceService.findYdDealPrice(model.getCity(), model.getDistrict(), model.getName()); +// if(ydDealPrice == null){ +// mav.setModel(BasicModel.buildOKModel().setMsg("该小区暂不支持估价").markNoData()); +// return mav; +// }else{ +// evaluation = evaluationService.ydEvaluate(district, buildYdEvaluation(model, request), model.getCalDate(),ydDealPrice.getPrice()); +// //近30天的涨跌幅:取最近1个月至3个月之间最近的那期价格和当前均价相比 +// Date end = DateUtil.getDateOfLastMonth(new Date(),-1); +// Date start = DateUtil.getDateOfLastMonth(new Date(),-3); +// List ydDealPrices = ydDealPriceService.findYdDealPrices(model.getCity(), model.getDistrict(), model.getName(),start,end); +// if(ydDealPrices!=null && ydDealPrices.size()>0){ +// oneMonthMarkup = (double)(ydDealPrices.get(0).getPrice()-ydDealPrice.getPrice())/ydDealPrice.getPrice(); +// } +// } +// }else{ evaluation = evaluationService.evaluate(district, buildYdEvaluation(model, request), model.getCalDate()); - } +// } }else{ evaluation = evaluationService.evaluate(district, buildEvaluation(model, request), model.getCalDate()); } @@ -1163,18 +1163,18 @@ public class FshController { private Map buildDistrictAvgZouShiByMonth(DistrictModel model,DistrictKey districtKey, Date end, Date start) { Map> avgPriceDetailMap = new HashMap>(); - if(model.isYouDa() && "上海".equals(model.getCityName())){ - List list = ydDealPriceService.findYdDealPrices(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end); - for (YdDealPrice stat : list) { - String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM"); - List prices = avgPriceDetailMap.get(ws); - if(null == prices){ - prices = new ArrayList(); - avgPriceDetailMap.put(ws, prices); - } - prices.add(stat.getPrice()); - } - }else{ +// if(model.isYouDa() && "上海".equals(model.getCityName())){ +// List list = ydDealPriceService.findYdDealPrices(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end); +// for (YdDealPrice stat : list) { +// String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM"); +// List prices = avgPriceDetailMap.get(ws); +// if(null == prices){ +// prices = new ArrayList(); +// avgPriceDetailMap.put(ws, prices); +// } +// prices.add(stat.getPrice()); +// } +// }else{ List list = districtStatService.findDistrictPrice(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end); for (DistrictStat stat : list) { String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM"); @@ -1185,7 +1185,7 @@ public class FshController { } prices.add(stat.getAvgPrice()); } - } +// } Map districtMap = new HashMap(); if(null != avgPriceDetailMap){ diff --git a/projects/fangjia-open/src/main/webapp/static/js/report.js b/projects/fangjia-open/src/main/webapp/static/js/report.js index b61a6e2d1..f3d833ccf 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/report.js +++ b/projects/fangjia-open/src/main/webapp/static/js/report.js @@ -362,7 +362,7 @@ $('.formal_btn').on('click', function () { }); function clickFormalReport(){ - $.prompt('姝e紡鎶ュ憡灏嗗敖蹇偖瀵勭粰鎮紝璇锋敞鎰忔煡鏀讹紝璋㈣阿锛佸鏈夌枒闂彲涓庢垜浠仈绯伙細62392317', '璇峰~鍐欐寮忔姤鍛婇偖瀵勫湴鍧�', function (value) { + $.prompt('姝e紡鎶ュ憡灏嗗敖蹇偖瀵勭粰鎮紝璇锋敞鎰忔煡鏀讹紝璋㈣阿锛� ', '璇峰~鍐欐寮忔姤鍛婇偖瀵勫湴鍧�', function (value) { var address = $(".modal-text-input").val(); if(address==""){ $.alert('鍦板潃涓嶈兘涓虹┖'); diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index f16637105..eaab26cb0 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -37,14 +37,13 @@

${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("鍙�")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}鍙�${fjtEvaluation.roomNum!}瀹� 閲嶆柊璇勪及 >

*璇风‘璁や笌浜ц瘉鍦板潃涓�鑷�

- <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0> -

璇勪及鎬讳环锛�<#if fjtEvaluation??> - ${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

-

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.inquirePrice gt 0>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

+ <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0> +

璇勪及鎬讳环锛�<#if fjtEvaluation??><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

+

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

<#if role?? && role==17 && loanInfoMap??>

鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

鍙捶鎴愭暟锛�${loanInfoMap.loan_ratio!}鎴�

-

鍙捶閲戦锛�${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

+

鍙捶閲戦锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

璐锋鍒╃巼锛�4.5%

<#else> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index dae53f491..af56de431 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -139,7 +139,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { query = new BasicDBObject() query.append("weekend", weekend) query.append("cat", "city") - cityCount = SQMETRE_RENT_COLL.count(query) + cityCount = SQMETRE_RENT_COLL.count(query) //本月区域平米租金计算量 query = new BasicDBObject() query.append("weekend", weekend) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 6ef05438e..3b92e612c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_all; def rows = [] def headers = [] for (def city : citys) { @@ -69,7 +69,7 @@ class BigdataBuildingBatch extends BaseBatch{ q.put("city", city) q.put("status", 1) // q.put("name","辽源一村") - def dbcursor = LOU_DONG_STAT.find(q) + def dbcursor = LOU_DONG_STAT.find(q).batchSize(5000) def map = [:] def set = new HashSet() for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 1000b157c..066318e29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -28,7 +28,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index b54b97e70..fafe61ba9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient @@ -27,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -35,7 +36,8 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -43,18 +45,19 @@ class BigdataCommunityBatch extends BaseBatch{ //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + println "starting 。。。。。。。。。" for(def city : citys){ + index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) - println city +" 小区量:"+dists.size() def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) - println city +" "+rets.size() - + println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } @@ -105,7 +108,6 @@ class BigdataCommunityBatch extends BaseBatch{ if(ret){ rets << ret } - } catch (Exception e) { e.printStackTrace() } finally { @@ -173,38 +175,25 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa101" ret << "" if(dist.address){ - if(dist.address.toString().contains("弄")){ - String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) - if(number){ - ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) - ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size()) + def jsonObj = null + if(dist.address){ + jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) + } + if(jsonObj && jsonObj.result && jsonObj.result.street){ + if(jsonObj.result.street){ + ret << jsonObj.result.street }else{ ret << "" + } + + if(jsonObj.result.streetNumber){ + ret << jsonObj.result.streetNumber + }else{ ret << "" } }else{ - def jsonObj = null - if(dist.address){ -// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) - jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) - } - if(jsonObj && jsonObj.street){ - if(jsonObj.street){ - ret << jsonObj.street - }else{ - ret << "" - } - - if(jsonObj.streetNumber){ - ret << jsonObj.streetNumber - }else{ - ret << "" - } - - }else{ - ret << "" - ret << "" - } + ret << "" + ret << "" } }else{ ret << "" @@ -220,10 +209,9 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "" ret << 0 }catch(Exception e){ - println dist.city + " "+dist.name + println dist.city + " "+dist.name+" error" e.printStackTrace() } - return ret; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 97f0a5bee..789603d29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap @@ -32,8 +33,11 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ +// def startDate = 202206 +// def endDate = 202208 + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{ DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -52,18 +57,23 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) - def citys = CcbCommonUtils.citys; -// def ccbCitys = CcbCommonUtils.citys; - - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } +// def citys = CcbCommonUtils.citys; + def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) + if(args && args.size()>=2){ + curDate = args[1].toString().toInteger() } + +// for(def city : citys){ +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } +// } def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 for(def city : citys){ + index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def ppeDists = [:] @@ -79,7 +89,7 @@ class BigdataDealBatch extends BaseBatch{ def trades = getCityData(city, statMap, ppeDists) def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") rows.addAll(rets) - println city +" 成交共:"+rets.size() + println "【${index}】"+city +" 成交共:"+rets.size() def cityDatas = [] cityDatas << headers @@ -91,10 +101,8 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成" - - + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + println "txt 生成 " + rows.size() } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -102,6 +110,9 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) +// def lianjiaPriceMap = findThirdPartyPrice("上海") + def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') + def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) for(def i=0;i 3000) { + price = db.lianjia_esf_price + } + if (!price && db.beike_esf_price && db.beike_esf_price > 3000) { + price = db.beike_esf_price + } + if(price){ + map.put(db.region+"##"+db.name, price) + } + + } + return map + } + + def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){ def ret = [] def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; } - ret << cityCodeDatas.get(trade.city) - ret << trade._id.toString() - ret << trade.city - def data = distCodeMap.get(trade.fjRegion) - if(data){ - ret << data.district_code - ret << data.district - }else{ - ret << "" - ret << "" - } - def street = "" - def streetNumber = "" - if(trade.address){ - if(trade.address.toString().contains("弄")){ - String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) - if(number){ - street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) - streetNumber = number + def price = lianjiaPriceMap.get(obj.region+"##"+obj.name) + if(price){ + def mod = java.lang.Math.abs((trade.price-price)*100/price) + if(mod<=10){ + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << trade.city + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" } - } - if(!street){ - def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) - if(jsonObj && jsonObj.searchAddress){ - if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + def street = "" + def streetNumber = "" + if(trade.address){ + if(trade.address.toString().contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) streetNumber = number } - }else{ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber } + if(!street){ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } + } } - } - } - if(!street){ - if(obj.address){ - if(obj.address.toString().contains("弄")){ - String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) - if(number){ - street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) - streetNumber = number + if(!street){ + if(obj.address){ + if(obj.address.toString().contains("弄")){ + String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) + if(number){ + street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) + if(jsonObj && jsonObj.searchAddress){ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } } + } + + ret << street + ret << streetNumber + + + if(obj){ + ret << obj._id.toString() + ret << trade.fjName + } + + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) + ret << ""//期数 + String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) + if(roomNum && roomNum.contains("0")){ + ret << roomNum.substring(0, roomNum.lastIndexOf("0")) }else{ - def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) - if(jsonObj && jsonObj.searchAddress){ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber - } + ret << trade.floor } + + ret << "" + ret << trade.height + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.price + ret << trade.totalPrice * 10000 + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "99" + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.totalPrice * 10000 + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") } } - ret << street - ret << streetNumber - - - if(obj){ - ret << obj._id.toString() - ret << trade.fjName - } - - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 - ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) - ret << ""//期数 - String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) - if(roomNum && roomNum.contains("0")){ - ret << roomNum.substring(0, roomNum.lastIndexOf("0")) - }else{ - ret << trade.floor - } - - ret << "" - ret << trade.height - def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 - def buildType = '' - if(arrs){ - buildType = arrs[0] - } - ret << buildType - ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) - ret << trade.area - ret << "" - ret << getHouseRoom(buildType, trade.room, trade.hall) - ret << trade.room - ret << trade.hall - ret << "" - ret << trade.toilet - ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) - ret << "" - ret << CcbCodeUtils.getDirectionCode(trade.direction) - ret << CcbCodeUtils.getFitmentCode(trade.fitment) - ret << "" - ret << trade.price - ret << trade.totalPrice * 10000 - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") - ret << "02" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "99" - - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << trade.totalPrice * 10000 - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") return ret; } @@ -289,7 +330,7 @@ class BigdataDealBatch extends BaseBatch{ } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() - ret << trade.city + ret << CcbCommonUtils.getCcbCity(trade.city) if(data){ ret << data.district_code ret << data.district @@ -543,11 +584,14 @@ class BigdataDealBatch extends BaseBatch{ def getDistrictStatByCity(city, weekend){ def q = new BasicDBObject() q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) + q.put("weekend", new BasicDBObject('$gte', weekend)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1)) def map = new ConcurrentHashMap() for(def d : dbcursor){ - map.put(d.region+"@"+d.district_name, d.avg_price) + def price = map.get(d.region+"##"+d.district_name) + if(!price && d.value_type==2){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } } return map } @@ -569,10 +613,10 @@ class BigdataDealBatch extends BaseBatch{ def addDealPriceMatch(city){ - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) -// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) -// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) final CountDownLatch latch = new CountDownLatch(trades.size()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index c3affa2bd..e5272b8bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql + /** * 建行模型估值项目 * 区域数据同步 @@ -17,33 +19,74 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class BigdataDistrictBatch extends BaseBatch{ + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { runBatch(new BigdataDistrictBatch(), args); } @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; - def rows = [] - for(def city : citys){ - def datas = findDistrictCodeValue(city) - if(datas){ - rows.addAll(datas) - } - println city+" "+ datas.size() - } - - def allDatas = createDatas(rows) - - def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] - + def datas = readText("D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.txt") def rets = [] - rets << headers - rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); + def index = 0 + for(def line : datas){ + def arrs = line[0].split("\\|@\\|") + def sql = "select count(*) as count from BIGDATA_DISTRICT_GJ where distid='${arrs[0]}'"; + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + if(count==0){ + rets << line + } + } + writeTxt(rets, "D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ_add.txt") +// def citys = CcbCommonUtils.citys_all; +// def rows = [] +// for(def city : citys){ +// def datas = findDistrictCodeValue(city) +// if(datas){ +// rows.addAll(datas) +// } +// println city+" "+ datas.size() +// } +// +// def allDatas = createDatas(rows) +// +// def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] +// +// def rets = [] +// rets << headers +// rets.addAll(allDatas) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") +// FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } + + def writeTxt(def rows, def path){ + StringBuilder builder = new StringBuilder() + for(def row : rows){ + builder.append(row) + builder.append('\n') + } + org.apache.commons.io.FileUtils.writeStringToFile(new File(path), builder.toString(), "GBK") + } + private def readText(def fileName){ + File file = new File(fileName) + def ret = []; + //对文件内容的每一行进行分割处理,比较常用在处理csv文件 + file.splitEachLine(" ") { + def temp = it + ret << temp; + } + //转为Writable对象,可重定向输出 + file.asWritable() + return ret; + } + + def createDatas(rows){ def allRows = [] def index = 1 @@ -72,16 +115,16 @@ class BigdataDistrictBatch extends BaseBatch{ def findDistrictCodeValue(city){ def q = new BasicDBObject() - q.put("fangjia_city", city) - // q.put("district", new BasicDBObject('$ne',"市辖区")) - def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + q.put("fjw_city", city) +// def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + def dbcursor = CcbCommonUtils.FANGJIA_AREA_CODE.find(q) def lists = [] for(def db : dbcursor){ if(db.district_code){ def m = [:] m.id = db._id.toString() m.dist_name = db.district - m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") + m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fjw_district, "") m.city_id = db.city_code m.dist_code = db.district_code lists << m diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 6e1b76aa5..5b0edbae1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -67,20 +67,39 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { def index = 0 - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) + statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } } + + def statCity(def citys){ + def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1) + def houseCitys = [] + for(def d : datas){ + houseCitys << d['C'] + } + for(def city : citys){ + if(!houseCitys.contains(city)){ + def scity = CcbCommonUtils.specialCityMap.get(city) + if(!scity){ + println city + } + } + } + } + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() @@ -429,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{ if(!alias.contains(newName)){ alias << newName } - } if(alias && alias.size()>0){ @@ -538,7 +556,7 @@ class BigdataForsaleBatch extends BaseBatch{ } rowMap.houseId = seawater.id - rowMap.city = city + rowMap.city = CcbCommonUtils.getCcbCity(city) //rowMap.region =seawater.region //rowMap.address = seawater.address diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index 39a26972b..08653a46a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{ @Override public void process(Object args) { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def citys = CcbCommonUtils.citys; -// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" - def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) - def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) -// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) - def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) - def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) -// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) + def citys = CcbCommonUtils.citys_252 + + +// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1) +// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1) +// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1) +// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1) + + def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1) def rets = [] // def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // rets.addAll(rows1) - - def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") - rets.addAll(rows1) - - def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") - rets.addAll(rows2) // -// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") +// rets.addAll(rows2) +// +// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") // rets.addAll(rows3) - - def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") - rets.addAll(rows4) - - def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") - rets.addAll(rows5) +//// +// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") +// rets.addAll(rows4) // -// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") -// rets.addAll(rows6) + def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") + rets.addAll(rows6) - def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] + +// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") +// rets.addAll(rows5) +// + + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat" if(CommonDatautils.isWindow()){ @@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{ }else if(type =="丧葬服务"){ ret << "ob101" }else if(type =="学校"){ - if(d['F'].contains("中学")){ + if(d['B'].contains("中学")){ ret << "ed138" - }else if(d['F'].contains("小学")){ + }else if(d['B'].contains("小学")){ ret << "ed114" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index e4f47e2fd..46aa81fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,12 +16,16 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] -// public static def citys = ["武汉"] + public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] + public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] - public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") +// public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") + + public static final DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") + public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") @@ -34,39 +38,76 @@ class CcbCommonUtils { public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"] + public static DSCollection dataStoreConnection = null; public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def getRunningCitys(def args){ + def citys = [] + println "args 参数:"+args.toString() + if(args){ + def countStr = args[0] + if(countStr=="344"){ + citys = CcbCommonUtils.citys_all + }else if(countStr=="92"){ + citys = CcbCommonUtils.citys_92 + }else if(countStr=="252"){ + citys = CcbCommonUtils.citys_252 + } + }else{ + citys = CcbCommonUtils.citys_92 + } + println "运行城市数量:"+citys.size() + return citys; + } + + public static def findCityCodeValue(){ - def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] + def cityCodeMap = [:] + FANGJIA_AREA_CODE.find().each {it-> + def code = cityCodeMap.get(it.fjw_city) + if(!code){ + cityCodeMap.put(it.fjw_city, it.city_code.toString()) + } + } +// def oldCityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] return cityCodeMap; } + public static def getCcbCity(def city){ + if(city && specialCityMap.containsKey(city)){ + return specialCityMap.get(city) + } + return city; + } + public static def findAllDistrictCodeValue(city){ def q = new BasicDBObject() - q.put("fangjia_city", city) + q.put("fjw_city", city) //q.put("district", new BasicDBObject('$ne',"市辖区")) - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - map.put(db.fangjia_district, db) + map.put(db.fjw_district, db) } return map; } public static def findAllCityDistrictCodeValue(city){ def q = new BasicDBObject() - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - def distMap = map.get(db.fangjia_city) + def distMap = map.get(db.fjw_city) if(!distMap){ distMap = [:] - map.put(db.fangjia_city, distMap) + map.put(db.fjw_city, distMap) } - distMap.put(db.fangjia_district, db) + distMap.put(db.fjw_district, db) } return map; } @@ -376,7 +417,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","辽源一村") +// q.put("name","云山尚城") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 7083cb615..2a9ad549f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -11,7 +11,7 @@ class CcbFtpTestUtils { static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" - static def ftpPath = "/home/ccb/output/" + static def ftpPath = "/home/ccb_evaluation/output/" static def ftpPath_rh = "/home/ccb_rh/output/" @@ -70,6 +70,22 @@ class CcbFtpTestUtils { } + static def listFileNames(month){ + //上传到ftp + sftpChannel = reConnect(); + Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + return files + } + + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 74474aff6..124911f34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -3,11 +3,12 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jiangnanblank.SFTP import com.jcraft.jsch.ChannelSftp +import com.jcraft.jsch.SftpATTRS import org.apache.commons.codec.digest.DigestUtils class CcbFtpUtils { - static def ftpServer = "192.168.0.45" + static def ftpServer = "192.168.0.45"//192.168.0.45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" @@ -64,6 +65,13 @@ class CcbFtpUtils { return files } + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 6dcffbacf..a96478424 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -370,13 +370,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def month = 202207 + def citys = CcbCommonUtils.citys_92 + def month = 202209 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202208) + def rets = findPeGteThirdParty(city, 202210) def total = 0 println city +" "+rets.size() for(def m : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 94741174c..ed5ed5f03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -62,7 +62,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ ret << dist._id.toString() ret << dist.name ret << cityCodeDatas.get(dist.city) - ret << dist.city + ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district ret << ps.avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e229de8e5..22861ba1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -15,38 +15,42 @@ class ExportOracleData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null + static main(args) { def task = new ExportOracleData() - task.run() + task.run(args) } - def run() { -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] - //不会变 + //不会变 10.25 exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() + exportBigdataPoiData()// + exportBigdataDistrictData()// - exportGuidedPrice()//随人工改变 - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // //处理小区情况 - exportBigdataCommunity() - exportCommunityStaging() - exportCommunityBuileType() + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// - exportThirdPartyPrice2(lastMonth, new_dists) + exportThirdPartyPrice2(lastMonth, new_dists)// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 exportWebsiteRate(); - - - //计算结果相关 +// +// +// //计算结果相关 // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) // @@ -58,11 +62,11 @@ class ExportOracleData { // // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) - +// // def curDate = curMonth.toString()+"28" // SSHHelper jschCommand = new SSHHelper(); // System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); - //ok文件 +//// ok文件 // exportLastFile() } @@ -129,24 +133,28 @@ class ExportOracleData { def exportBigdataPoiData() { - def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.CITYID - data << r.DISTID - data << r.ID - data << r.POI_NAME - data << r.TYPE - data << r.TYPE_CODE - data << r.LATITUDE - data << r.LONGITUDE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; @@ -163,8 +171,9 @@ class ExportOracleData { } def exportCommunityStaging() { - def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + def allRows = [] + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -193,9 +202,9 @@ class ExportOracleData { // def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def allRows = [] -// for(def city : citys){ -// def cityid = cityCodeDataMap.get(city) - def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.CITYID @@ -208,7 +217,8 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); -// } + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" @@ -224,28 +234,31 @@ class ExportOracleData { } def exportBigdataDistrictData() { - - def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.LJXZQH - data << r.LJXZQHDM - data << r.XZQHDM - data << r.WORKDATE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -262,26 +275,30 @@ class ExportOracleData { def exportGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.DEALDATE - data << r.AVG_PRICE - data << r.GUIDING_PERSON - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -296,25 +313,29 @@ class ExportOracleData { } def exportCommunityGuidedPrice(def date) { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.AVGPRICE - data << r.SOURCETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -331,30 +352,33 @@ class ExportOracleData { def exportBigdataDeal(def dealDate, def new_dists) { def allRows = [] - if(new_dists){ - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" - def rets1 = dealData(sql1) + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) - def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where distid in ${newstr}" - def rets2 = dealData(sql2) - allRows.addAll(rets1) - allRows.addAll(rets2) - }else{ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; @@ -363,8 +387,6 @@ class ExportOracleData { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } def headers = [] -//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DEAL_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) @@ -497,25 +519,6 @@ class ExportOracleData { def exportThirdPartyPrice2(price_date, new_dists) { - def allRows = [] - if(new_dists) { - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + - "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + - "price_date=${price_date} and distid not in ${newstr} "; - def rets1 = getThirdPartyDatas(sql1) - allRows.addAll(rets1) - - def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + - "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + - "distid in ${newstr} "; - def rets2 = getThirdPartyDatas(sql2) - allRows.addAll(rets2) - }else{ - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; - allRows = getThirdPartyDatas(sql) - } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -523,9 +526,37 @@ class ExportOracleData { // fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] -//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) } @@ -552,53 +583,58 @@ class ExportOracleData { } def exportBigdataCommunity(){ - def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + - "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.DISTID - data << r.COMMID - data << r.DISTNAME - data << r.COMMNAME - data << r.ADDRESS - data << r.LATITUDE - data << r.LONGITUDE - data << r.LANDAREA - data << r.BUILDAREA - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << r.BUILDYEAR - data << r.HOUSEUSE - data << r.ZTS - data << r.WORKDATE - data << r.UPDATEFLAG - data << r.UPDATETIME - data << r.COMMTYPE - data << r.COMMALIAS - data << r.STREET - data << r.STREETNO - data << r.IFNEW - data << "" - data << r.REMARK - data << r.CREATETIME - data << r.SOURCECODE - data << r.XZJDB - data << r.LJXZQHDM - data << r.LJXZQH - data << r.STATUS - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", - //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", - //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) @@ -607,18 +643,15 @@ class ExportOracleData { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 for(def city : citys){ - def cityCode = cityCodeDataMap.get(city) - + def cityCode = cityCodeMap.get(city) def allRows = [] if(new_dists){ @@ -636,7 +669,13 @@ class ExportOracleData { allRows = getHouseDatas(sql) } if(allRows.size()>0){ - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } } @@ -778,8 +817,8 @@ class ExportOracleData { fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", - //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", - //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "AVM_EVALUATION_GJNB 数据文件输出完成......" @@ -993,55 +1032,55 @@ class ExportOracleData { // // for(def city : citys) { // def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; - def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMUNITY_ID - data << r.CITYID - data << r.CITY_NAME - data << r.DISTRICT_CODE - data << r.DISTRICT_NAME - data << r.COMMUNITY_NAME - data << r.PQ_NAME - data << r.BASE_PRICE - data << r.CONFIDENCE_LEVEL - data << r.CNT - data << r.MIN_PRICE - data << r.MEDIANPRICE - data << r.MEANPRICE - data << r.MAX_PRICE - data << r.TIME - data << r.FIRSTDATAT - data << r.THIRDDATA - data << r.RELATIVEERROR - data << r.SKEWNESS - data << r.PQ_PRICE_RANGE - data << r.DISTRICT_PRICE_RANGE - data << r.DESCRIPTION - data << r.PRIV_BASE_PRICE - data << r.IS_ARTTFICIAL - data << r.PRIV_FINAL_PRICE - data << r.CREATE_DATE - data << r.PRICE - data << r.PRICE_SOURCE - data << r.CASE_MD_PRICE - data << r.CASE_AVG_PRICE - data << r.INDEX_LABEL - data << r.MAIN_PROPERTY - data << r.COMMTYPE - data << r.NEW_DEAL_PRICE - data << r.NEW_DEAL_DATE - data << r.PRICE_MOM_RATIO - data << r.MAX_MOM_RATIO - data << r.PRICE_YOY_RATIO - data << r.SATUR_RATIO - data << r.NEIGHBOR_PRICE - allRows << data - }); - println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); // } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy new file mode 100644 index 000000000..431fc02c2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -0,0 +1,1171 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +import java.sql.ResultSet + +class ExportOracleDataTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null + + static main(args) { + def task = new ExportOracleDataTest() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + //不会变 10.25 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + + //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth)//无城市区分 + exportAVMEvaluationItemGJNB(curMonth)//无城市区分 + + exportCityPrice(curMonth)//无城市区分 + exportRegionPrice(curMonth)//无城市区分 + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 + exportCommunityBasePrice(curMonth)//无城市区分 + + exportCommunityPdescribeGj(curMonth)//无城市区分 + exportCommCaseDetailGj(curMonth)//无城市区分 + + def curDate = curMonth.toString()+"28" + // ok文件 + exportInOk() + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + + } + + + def exportEvaluationRateData() { + def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportWebsiteRate(){ + def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.DATA_SOURCE_MAP + data << r.WEIGHT + data << r.TYPE + data << r.SOURCECODE + data << r.CITYID + data << r.CITYNAME + data << r.CREATETIME + data << r.UPDATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_POI_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_POI_GJ 上传Ftp完成......" + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.MASTER_COMMID + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_STAGING_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_STAGING_GJ 上传Ftp完成......" + } + + def exportCommunityBuileType() { +// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "GUIDED_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "GUIDED_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DEAL_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DEAL_GJ 上传Ftp完成......" + } + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.CJJE + data << r.WORKDATE + allRows << data + }); + return allRows + } + + def exportThirdPartyPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + + "from third_party_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.ZHUGE_AVG_PRICE + data << r.ANJUKE_AVG_PRICE + data << r.FANG_AVG_PRICE + data << r.LIANJIA_AVG_PRICE + data << r.CITYHOUSE_AVG_PRICE + data << r.MAITIAN_AVG_PRICE + data << r.KE_AVG_PRICE + data << r.TAOFANG_AVG_PRICE + data << r.ZHONGYUAN_AVG_PRICE + data << r.TBS_AVG_PRICE + data << r.QFANG_AVG_PRICE + data << r.FANGTAN_AVG_PRICE + data << r.FANGDUODUO_AVG_PRICE + data << r.WIWJ_AVG_PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE.replaceAll(~'\\r', '') + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......" + } + + + def exportThirdPartyPrice2(price_date, new_dists) { + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + return allRows; + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" + } + + def exportBigdataForsale(publishDate, new_dists){ + //["北京","上海"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(yearMonth){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePriceGj(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" + } + + def exportCityPrice(month){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(month){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityPdescribeGj(month){ + +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +// +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; +// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + +// def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportOkFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpTestUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + } + } + def fileName = "/ccb_evaluation/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "统计文件长度ok文件 上传Ftp完成......" + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 7258934fa..b37ee5b94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -12,13 +12,14 @@ class ExportOracleEvaluationData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def citys = null static main(args) { def task = new ExportOracleEvaluationData() - task.run() + task.run(args) } - def run() { - + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 @@ -369,7 +370,7 @@ class ExportOracleEvaluationData { def exportCommCaseDetailGj(month){ - def citys = CcbCommonUtils.citys; +// def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 0715fc907..b28516075 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -27,9 +27,46 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() - task.addFjwPrice() +// task.addFjwPrice() +// task.addCcbPrice() + task.addMarketPrice() } + def addMarketPrice(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\拍卖南京.xlsx",{d-> + if(d['D']) { + d['K'] = d['D'] + }else if(d['H']){ + d['K'] = d['H'] + } + + }) + } + + def addCcbPrice(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d-> + if(d['B']){ + def cityId = cityCodeDataMap.get(d['B']) + def dist = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district") + if(dist){ + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209) + if(map){ + d['I'] = map.avg_price + d['J'] = map.calc_type + d['k'] = map.credit + } + } + +// def dist = CommonDatautils.findDistrictById(d['B']) +// if(dist){ +// d['J'] = dist.status +// } + } + }) + } + + def updatePrice(){ def index = 0 ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d-> @@ -174,6 +211,27 @@ class FjwPricePkCcbPrice { } + def getCommunityBasePriceGjByCommid(cityid, commid, yearMonth) { + def sql = "select AVG_PRICE, CALC_TYPE,CONFIDENCE from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} and commid='${commid}'"; + def map = [:] + sqlInstance236.eachRow(sql, { r -> + map.avg_price = r.AVG_PRICE + // 计算类型,1:人工估值 2:样本计算 3:神经网络 + if(r.CALC_TYPE){ + if(r.CALC_TYPE == 1){ + map.calc_type = "人工估值" + }else if(r.CALC_TYPE == 2){ + map.calc_type = "样本计算" + }else if(r.CALC_TYPE == 3){ + map.calc_type = "神经网络" + } + } + map.credit = r.CONFIDENCE + }); + return map + } + + def batchCreateFile(ret, fileName, headers, charset){ OutputStream out = new FileOutputStream(fileName); def sb = new StringBuffer(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index f839de5f5..6981c7149 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for (def city : citys) { @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) @@ -147,7 +147,7 @@ class GuiDingPriceBatch extends BaseBatch{ ret << dist._id.toString() ret << dist.name ret << cityCodeDatas.get(dist.city) - ret << dist.city + ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 3ac7f9a28..ed95e4cc6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -24,7 +24,7 @@ class StagingDistrictBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for(def city : citys){ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index bd932d6fd..e987ea5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,935 +45,996 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def url = "https://sh.anjuke.com" -//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") -//println s - -//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") -//for(def d : districts){ -// println d.name -//} - -//def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) -//for(def d : dists){ -// def o = CommonDatautils.findDistrictById(d._id.toString()) -// println o._id.toString() + " "+o.name -//} - -//def citys = CcbCommonUtils.citys; -//def rows = [] +////def url = "https://sh.anjuke.com" +////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") +////println s // -//for(def city : citys) { -// def data = CcbCommonUtils.findCityCodeValue() -// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +////def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") +////for(def d : districts){ +//// println d.name +////} // -// def count = 0 -// sqlInstance.execute(sql,{ -// count = it.count -// }); -// println city +" "+count -//} - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) -//} -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) -// } -//},2) - - - - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" - -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// def map = [:] -// map.put("land_area",d['D']) -// map.put("build_area",d['F']) -// map.put("greeningrate_area",d['H']) -// map.put("parkingcnt_area",d['J']) -// map.put("zts_area",d['L']) -// map.put("buildyear_area",d['N']) -// map.put("plotratio",d['P']) +////def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) +////for(def d : dists){ +//// def o = CommonDatautils.findDistrictById(d._id.toString()) +//// println o._id.toString() + " "+o.name +////} // -// dealMap.put(d['A'], map) -//} - -//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -////小区量 -//ExcelUtils.multiThreadingModify(fileName, { d -> -// def city = d['B'] -// if (city) { -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['P'] = total -// } -//},0) - -////小区字段覆盖率 -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def city = d['A'] -// if(city){ -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['AA'] = total -// -// def a = findEstateType(cityid, sqlInstance) -// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -// -// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -// -// def c = findFiledCount(cityid, 'address', sqlInstance) -// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -// -// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -// -// def e = findFiledCount(cityid, 'landarea', sqlInstance) -// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -// -// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -// -// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -// -// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -// -// def i = findFiledCount(cityid, 'latitude', sqlInstance) -// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -// -// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) -// println d -// } -//},1) - -//def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" -//def datas =ExcelUtils.read(fileName,0,1) -//def index = 0 -//def rows = [] -//for(def d : datas) { -//// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// if (count == 0) { -// index++ -// rows << d -// println index +" "+d -// } -//// if(rows.size()>20){ -//// break -//// } -//// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") -//// if(dists.size()==1){ -//// def dist = dists[0] -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// }else{ -//// for(def r :dists){ -//// if(d['B'].contains(r.region)){ -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// } -//// } -//// } -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") -//println "共查到:"+index - - -def index = 0 -//def commIdThirdMap = findAllThird(202207, sqlInstance236); -def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -//cityCodeDatas.each{city,cityId-> -// -// deleteThirdByCityId(cityId, 202207, sqlInstance236) -// println city + "完成" -//} - - -def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) -for(line in lines) { - def arrs = line.toString().split("\\|@\\|") - if(arrs){ - def sql = "delete from guided_price_gj where id='${arrs[0]}'" - sqlInstance236.executeUpdate(sql) - } -} - -def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ - def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" - sqlInstance236.executeUpdate(sql) -} -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> -// if (d['C']) { -// index ++T -// def cityId = cityCodeDatas.get(d['A']) -// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) -// if(sources.size()>1){ -// if(sources.contains("安居客") && sources.contains("58同城")){ -// sources.remove("58同城") -// }else if(sources.contains("链家") && sources.contains("贝壳")){ -// sources.remove("贝壳") -// } -// } -// if(sources.size()==1){ -// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId -// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) -// } -// } -//},0) - - -def findAllThird(price_date, def sqlInstance236){ - def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" - def map = [:] - sqlInstance236.eachRow(sql,{r-> - def sources = map.get(r.commid) - if(!sources){ - sources = [] - map.put(r.commid, sources) - } - sources << r.data_source - }); - return map -} - - -def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ - def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" - sqlInstance236.executeUpdate(sql) -} - -def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ - def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" - def sources = [] - sqlInstance236.eachRow(sql,{r-> - sources << r.data_source - }); - return sources -} - - -def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ - def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ - def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" - def count = 0 - sqlInstance.eachRow(sql,{r-> - if(r.dealunitprice == price){ - count++ - }else{ - println r.dealunitprice +" price:"+price - } - - }); - return count -} - - -def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ - def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") - def q = new BasicDBObject() - q.put("contractTime", date); -// q.put("area", new BasicDBObject('$gte', 20)) - q.put("fjCity",city); - q.put("fjRegion",region); - q.put("fjName",name); - q.put("area",area); - q.put("price", price) - def count = youda_esfhouse_coll.count(q) - return count; -} - - -//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202203) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['X'] = oldcount -// def casecount = testMap.get(d['C']) -// d['Y'] = casecount -// println d -// } -//},0) -//def rows = findTradePpeData(sqlInstance236) -//// rows.unique() -//def map=findTradeDetail(sqlInstance236,null) -//def datas=[] -//map.each{key,row-> -// if(!rows.contains(key)){ -// datas< -// def rows = findTradePpeData(sqlInstance236, 202203,code) -// def alltrades=findTradeDetail(sqlInstance236,code) -// alltrades.each{it-> -// def commid=it[0] -// if(!rows.contains(commid)){ -// datas< -// if(d['I'] && d['K']){ -// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") -// if(dist){ -// d['S'] = dist._id.toString() -// } -// } -//},0) - -//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> -// if(d['A']){ -// def obj = CommonDatautils.findDistrictById(d['A']) -// if(obj && obj.name.contains("一期")){ -// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) -// if(newName){ -// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") -// if(masterDist && masterDist.name_cat==0){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 9) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", obj._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// d['D'] = "主名不存在" -// } -// } -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> -// def flag = d['M'] -// if(flag==0.0){ -// if(d['D']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['D'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -// } -// -//},0) - -//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> -// if(d['A']){ -//// def region = d['B'] -//// def newRegion = RegionData.formatRegion(d['A'], region) -//// if(newRegion) { -//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println d -// } -//// } -// } -//},0) - - -//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> -// if(d['A']){ -//// def region = d['B'] -//// def newRegion = RegionData.formatRegion(d['A'], region) -//// if(newRegion) { -//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println d -// } -//// } -// } -//},0) - - - - -//def citys = CcbCommonUtils.citys -//def rets = [] -//for(def city : citys){ -// def count = CommonDatautils.findSeaweeds(city, "district", 1) -// rets << [city, count] -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") - -//def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] -//for(def city : citys){ -// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) -// def index = 0 -// for(def dist : districts){ -// if(dist.name_cat && dist.name_cat==1){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 0) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// } -// } -// println city +" "+index -//} - - -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> -// if(d['C']){ -//// def udata = new BasicDBObject() -//// udata.put("name_cat", 0) -//// udata.put("m_date",new Date()) +////def citys = CcbCommonUtils.citys; +////def rows = [] //// -//// def q = new BasicDBObject() -//// q.put("_id", new ObjectId(d['C'])) -//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// def count = findDistCount(d['C'], sqlInstance); -// if(count==0){ -// println d['C'] -// } -// } -//},0) - - +////for(def city : citys) { +//// def data = CcbCommonUtils.findCityCodeValue() +//// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +//// +//// def count = 0 +//// sqlInstance.execute(sql,{ +//// count = it.count +//// }); +//// println city +" "+count +////} +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(d['C']){ +//// d['Q'] = dealMap.get(d['C']) +//// } +////},2) +// +// +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" +// +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// def map = [:] +//// map.put("land_area",d['D']) +//// map.put("build_area",d['F']) +//// map.put("greeningrate_area",d['H']) +//// map.put("parkingcnt_area",d['J']) +//// map.put("zts_area",d['L']) +//// map.put("buildyear_area",d['N']) +//// map.put("plotratio",d['P']) +//// +//// dealMap.put(d['A'], map) +////} +// +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//////小区量 +////ExcelUtils.multiThreadingModify(fileName, { d -> +//// def city = d['B'] +//// if (city) { +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['P'] = total +//// } +////},0) +// +//////小区字段覆盖率 +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def city = d['A'] +//// if(city){ +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['AA'] = total +//// +//// def a = findEstateType(cityid, sqlInstance) +//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +//// +//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +//// +//// def c = findFiledCount(cityid, 'address', sqlInstance) +//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +//// +//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +//// +//// def e = findFiledCount(cityid, 'landarea', sqlInstance) +//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +//// +//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +//// +//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +//// +//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +//// +//// def i = findFiledCount(cityid, 'latitude', sqlInstance) +//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +//// +//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +//// println d +//// } +////},1) +// +////def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" +////def datas =ExcelUtils.read(fileName,0,1) +////def index = 0 +////def rows = [] +////for(def d : datas) { +////// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// if (count == 0) { +//// index++ +//// rows << d +//// println index +" "+d +//// } +////// if(rows.size()>20){ +////// break +////// } +////// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") +////// if(dists.size()==1){ +////// def dist = dists[0] +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// }else{ +////// for(def r :dists){ +////// if(d['B'].contains(r.region)){ +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// } +////// } +////// } +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") +////println "共查到:"+index +// +// //def index = 0 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ +////def commIdThirdMap = findAllThird(202207, sqlInstance236); +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +////cityCodeDatas.each{city,cityId-> +//// +//// deleteThirdByCityId(cityId, 202207, sqlInstance236) +//// println city + "完成" +////} // -// def alias = [] -// if(d['H'] && d['H']!=dist.name){ -// alias << d['H'] -// if(d['H'].contains("·")){ -// alias << d['H'].replace("·", "") -// } -// if(dist.alias){ -// for(def aliasName : dist.alias){ -// if(!alias.contains(aliasName)){ -// alias << aliasName -// } -// } -// } -// } // -// def address = null -// if(d['I']){ -// address = AddressUtil.formatAddress(dist.city, d['I']) -// } -// -// Long lng = null -// Long lat = null -// Long lat2 = null -// Long lng2 = null -// if(d['J']){ -// lng = d['J'].toString().toDouble() * 10000000000l; -// lat = d['K'].toString().toDouble() * 10000000000l; -// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) -// if(json != null && json.containsKey("x")) { -// lat2 = json.get("y") * 10000000000l -// lng2 = json.get("x") * 10000000000l -// } -// } -// -// def region = null -// if(dist.region!=d['F']){ -// region = dist.region -// } -// -// def udata = new BasicDBObject() -// if(region){ -// udata.put("region", region) -// } -// -// if(address){ -// udata.put("address", address) -// } -// if(lat2){ -// udata.put("lng", lng) -// udata.put("lat", lat) -// udata.put("lng2", lng2) -// udata.put("lat2", lat2) -// } -// -// if(alias){ -// udata.put("alias", alias) -// } -// if(udata){ -// index++ -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) -// println index+ " "+dist.city +" "+dist.name+" "+udata -// } -// -// } +//def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) +//for(line in lines) { +// def arrs = line.toString().split("\\|@\\|") +// if(arrs){ +// def sql = "delete from guided_price_gj where id='${arrs[0]}'" +// sqlInstance236.executeUpdate(sql) // } +//} // -//},0) - - - -//def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) -// if(obj && obj.alias){ -// d['H'] = obj.alias.join(" ") -// } -//// def citycode = d['A'] -//// cityCodeDataMap.each {k,v-> -//// if(citycode.contains(v)){ -//// d['D'] = k +//def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) +//} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> +//// if (d['C']) { +//// index ++T +//// def cityId = cityCodeDatas.get(d['A']) +//// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) +//// if(sources.size()>1){ +//// if(sources.contains("安居客") && sources.contains("58同城")){ +//// sources.remove("58同城") +//// }else if(sources.contains("链家") && sources.contains("贝壳")){ +//// sources.remove("贝壳") +//// } +//// } +//// if(sources.size()==1){ +//// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId +//// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) //// } //// } -//},0) - - - - -//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" -//def datas = ExcelUtils.read(fileName, 0, 1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) +////},0) +// +// +//def findAllThird(price_date, def sqlInstance236){ +// def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" +// def map = [:] +// sqlInstance236.eachRow(sql,{r-> +// def sources = map.get(r.commid) +// if(!sources){ +// sources = [] +// map.put(r.commid, sources) +// } +// sources << r.data_source +// }); +// return map //} -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> -// if(d['A']){ -// d['W'] = dealMap.get(d['A']) -// } -//},0) - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def deleteIndex = 0 -//ExcelUtils.modify(fileName,{d-> -// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ -// def dist = CommonDatautils.findDistrictById(d['D']) -// def city = dist.city -// def region = RegionData.formatRegion(city, d['Q']) -// if(region && dist.region!=region){ -// println city +" new:"+region +" old:"+d['C'] -// def udata = new BasicDBObject() -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// deleteIndex++ -// udata.put("status", 1) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// }else{ -// updateIndex++ -// udata.put("region", region) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// } -// }else{ -// println d['D']+" 不存在" -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// if(d['K']){ -// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) -// def address = null -// if(formatAddress && formatAddress.endsWith("号")) { -// address = formatAddress -// } // // -// def data = new BasicDBObject(); -// data.put("m_date", new Date()) -// if(d['M']){ -// long glat = d['M'].toString().toDouble() * 10000000000L -// long glng = d['L'].toString().toDouble() * 10000000000L -// data.put("lat",glat) -// data.put("lng",glng) -// -// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) -// if(baidu){ -// long blng = baidu.x * 10000000000L -// long blat = baidu.y * 10000000000L -// data.put("lat2",blat) -// data.put("lng2",blng) -// } -// -// } -// -// if(address){ -// data.put("address", address) -// } -// index++ -// println index +" "+d['B'] + " "+data -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) -// } -// } -// } -//}) - - -def findDealCount(sqlInstance, def startDate, def endDate){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate - map.put(r.cityname, r.count) - }); - return map -} - - -def findCaluateCityPPE(def sqlInstance, def yearMonth){ - def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; - def map = [:] - sqlInstance.eachRow(sql,{r-> - map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) - }); - return map -} - - -def findTradePpeData(def sqlInstance){ - def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; - def rows = new HashSet() - sqlInstance.eachRow(sql,{r-> - rows< - def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate - def row=[] - row< - count = r.count - }); - return count -} - -def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - - -def findDistCount(commid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def delIndex = 0 -//def cityMap = [:] -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(!d['R']){ -// -// def obj = CommonDatautils.findDistrictById(d['D']) -// if(obj){ -// def ranges = cityMap.get(obj.city) -// if(!ranges){ -// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) -// ranges = [] -// for(def r : dbcursor){ -// ranges << r -// } -// cityMap.put(obj.city, ranges) -// } -// -// if(ranges){ -// def lng = obj.lng/10000000000.0 -// def lat = obj.lat/10000000000.0 -// -// def region = null -// for(def r : ranges){ -// try{ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// def regionStr = r.region -// region = RegionData.formatRegion(obj.city, regionStr) -// break -// } -// }catch(Exception e){ -// println e.getMessage() -// } -// -// } -// if(region && region!=obj.region){ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name -// }else if(region && region==obj.region){ -// println "[same] "+obj.city +" "+obj.name -// }else{ -// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// delIndex++ -// println delIndex +" [delete] "+obj.city +" "+obj.name -// } -// }else{ -// println obj.city +" 没有此城市边框信息" -// } -// -// }else{ -// println d['D']+" 没找到该小区" -// } -// -// } -//},0); - - - -//def city = "杭州" -//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) -//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) -//def ranges = [] -//for(def d : dbcursor){ -// ranges << d +//def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) //} -//def index =0 -//def updateIndex = 0 -//def delIndex = 0 -//for(def dist : dists){ -//// if(dist.region=="江干"){ -// def lat = dist.lat/10000000000.0 -// def lng = dist.lng/10000000000.0 // -// def regionStr =null -// def region = null -// for(def r : ranges){ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// regionStr = r.region -// region = RegionData.formatRegion(dist.city, regionStr) -// break -// } -// } -// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) -// if(regionStr && region!=dist.region){ +//def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// def sources = [] +// sqlInstance236.eachRow(sql,{r-> +// sources << r.data_source +// }); +// return sources +//} // -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// delIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// println delIndex+" [del]"+region +" "+dist.name -// }else{ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name -// } +// +//def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ +// def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ +// def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// if(r.dealunitprice == price){ +// count++ // }else{ -// index++ -// println index+" [没找到]"+dist.name +// println r.dealunitprice +" price:"+price // } +// +// }); +// return count +//} +// +// +//def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ +// def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") +// def q = new BasicDBObject() +// q.put("contractTime", date); +//// q.put("area", new BasicDBObject('$gte', 20)) +// q.put("fjCity",city); +// q.put("fjRegion",region); +// q.put("fjName",name); +// q.put("area",area); +// q.put("price", price) +// def count = youda_esfhouse_coll.count(q) +// return count; +//} +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +////def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") +////def testMap = findCaluateCityPPE(sqlInstance236, 202203) +////ExcelUtils.modify(fileName,{d-> +//// if(d['C']){ +//// def oldcount = dealMap.get(d['C']) +//// d['X'] = oldcount +//// def casecount = testMap.get(d['C']) +//// d['Y'] = casecount +//// println d //// } +////},0) +////def rows = findTradePpeData(sqlInstance236) +////// rows.unique() +////def map=findTradeDetail(sqlInstance236,null) +////def datas=[] +////map.each{key,row-> +//// if(!rows.contains(key)){ +//// datas< +//// def rows = findTradePpeData(sqlInstance236, 202203,code) +//// def alltrades=findTradeDetail(sqlInstance236,code) +//// alltrades.each{it-> +//// def commid=it[0] +//// if(!rows.contains(commid)){ +//// datas< +//// if(d['I'] && d['K']){ +//// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") +//// if(dist){ +//// d['S'] = dist._id.toString() +//// } +//// } +////},0) +// +////0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> +//// if(d['A']){ +//// def obj = CommonDatautils.findDistrictById(d['A']) +//// if(obj && obj.name.contains("一期")){ +//// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) +//// if(newName){ +//// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") +//// if(masterDist && masterDist.name_cat==0){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 9) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", obj._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// d['D'] = "主名不存在" +//// } +//// } +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> +//// def flag = d['M'] +//// if(flag==0.0){ +//// if(d['D']){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", new ObjectId(d['D'])) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// println index +//// } +//// } +//// +////},0) +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +// +// +////def citys = CcbCommonUtils.citys +////def rets = [] +////for(def city : citys){ +//// def count = CommonDatautils.findSeaweeds(city, "district", 1) +//// rets << [city, count] +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") +// +////def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] +////for(def city : citys){ +//// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) +//// def index = 0 +//// for(def dist : districts){ +//// if(dist.name_cat && dist.name_cat==1){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 0) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// } +//// } +//// println city +" "+index +////} +// +// +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> +//// if(d['C']){ +////// def udata = new BasicDBObject() +////// udata.put("name_cat", 0) +////// udata.put("m_date",new Date()) +////// +////// def q = new BasicDBObject() +////// q.put("_id", new ObjectId(d['C'])) +////// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// def count = findDistCount(d['C'], sqlInstance); +//// if(count==0){ +//// println d['C'] +//// } +//// } +////},0) +// +// +////def index = 0 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// +//// def alias = [] +//// if(d['H'] && d['H']!=dist.name){ +//// alias << d['H'] +//// if(d['H'].contains("·")){ +//// alias << d['H'].replace("·", "") +//// } +//// if(dist.alias){ +//// for(def aliasName : dist.alias){ +//// if(!alias.contains(aliasName)){ +//// alias << aliasName +//// } +//// } +//// } +//// } +//// +//// def address = null +//// if(d['I']){ +//// address = AddressUtil.formatAddress(dist.city, d['I']) +//// } +//// +//// Long lng = null +//// Long lat = null +//// Long lat2 = null +//// Long lng2 = null +//// if(d['J']){ +//// lng = d['J'].toString().toDouble() * 10000000000l; +//// lat = d['K'].toString().toDouble() * 10000000000l; +//// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +//// if(json != null && json.containsKey("x")) { +//// lat2 = json.get("y") * 10000000000l +//// lng2 = json.get("x") * 10000000000l +//// } +//// } +//// +//// def region = null +//// if(dist.region!=d['F']){ +//// region = dist.region +//// } +//// +//// def udata = new BasicDBObject() +//// if(region){ +//// udata.put("region", region) +//// } +//// +//// if(address){ +//// udata.put("address", address) +//// } +//// if(lat2){ +//// udata.put("lng", lng) +//// udata.put("lat", lat) +//// udata.put("lng2", lng2) +//// udata.put("lat2", lat2) +//// } +//// +//// if(alias){ +//// udata.put("alias", alias) +//// } +//// if(udata){ +//// index++ +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +//// println index+ " "+dist.city +" "+dist.name+" "+udata +//// } +//// +//// } +//// } +//// +////},0) +// +// +// +////def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) +//// if(obj && obj.alias){ +//// d['H'] = obj.alias.join(" ") +//// } +////// def citycode = d['A'] +////// cityCodeDataMap.each {k,v-> +////// if(citycode.contains(v)){ +////// d['D'] = k +////// } +////// } +////},0) +// +// +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" +////def datas = ExcelUtils.read(fileName, 0, 1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> +//// if(d['A']){ +//// d['W'] = dealMap.get(d['A']) +//// } +////},0) +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def deleteIndex = 0 +////ExcelUtils.modify(fileName,{d-> +//// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ +//// def dist = CommonDatautils.findDistrictById(d['D']) +//// def city = dist.city +//// def region = RegionData.formatRegion(city, d['Q']) +//// if(region && dist.region!=region){ +//// println city +" new:"+region +" old:"+d['C'] +//// def udata = new BasicDBObject() +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// deleteIndex++ +//// udata.put("status", 1) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// }else{ +//// updateIndex++ +//// udata.put("region", region) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// } +//// }else{ +//// println d['D']+" 不存在" +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// if(d['K']){ +//// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) +//// def address = null +//// if(formatAddress && formatAddress.endsWith("号")) { +//// address = formatAddress +//// } +//// +//// +//// def data = new BasicDBObject(); +//// data.put("m_date", new Date()) +//// if(d['M']){ +//// long glat = d['M'].toString().toDouble() * 10000000000L +//// long glng = d['L'].toString().toDouble() * 10000000000L +//// data.put("lat",glat) +//// data.put("lng",glng) +//// +//// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) +//// if(baidu){ +//// long blng = baidu.x * 10000000000L +//// long blat = baidu.y * 10000000000L +//// data.put("lat2",blat) +//// data.put("lng2",blng) +//// } +//// +//// } +//// +//// if(address){ +//// data.put("address", address) +//// } +//// index++ +//// println index +" "+d['B'] + " "+data +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) +//// } +//// } +//// } +////}) +// +// +//def findDealCount(sqlInstance, def startDate, def endDate){ +// def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate +// map.put(r.cityname, r.count) +// }); +// return map +//} +// +// +//def findCaluateCityPPE(def sqlInstance, def yearMonth){ +// def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; +// def map = [:] +// sqlInstance.eachRow(sql,{r-> +// map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) +// }); +// return map +//} +// +// +//def findTradePpeData(def sqlInstance){ +// def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; +// def rows = new HashSet() +// sqlInstance.eachRow(sql,{r-> +// rows< +// def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate +// def row=[] +// row< +// count = r.count +// }); +// return count +//} +// +//def findCount(cityid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +// +//def findDistCount(commid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findFiledCount(cityid, field, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def delIndex = 0 +////def cityMap = [:] +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(!d['R']){ +//// +//// def obj = CommonDatautils.findDistrictById(d['D']) +//// if(obj){ +//// def ranges = cityMap.get(obj.city) +//// if(!ranges){ +//// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) +//// ranges = [] +//// for(def r : dbcursor){ +//// ranges << r +//// } +//// cityMap.put(obj.city, ranges) +//// } +//// +//// if(ranges){ +//// def lng = obj.lng/10000000000.0 +//// def lat = obj.lat/10000000000.0 +//// +//// def region = null +//// for(def r : ranges){ +//// try{ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// def regionStr = r.region +//// region = RegionData.formatRegion(obj.city, regionStr) +//// break +//// } +//// }catch(Exception e){ +//// println e.getMessage() +//// } +//// +//// } +//// if(region && region!=obj.region){ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name +//// }else if(region && region==obj.region){ +//// println "[same] "+obj.city +" "+obj.name +//// }else{ +//// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// delIndex++ +//// println delIndex +" [delete] "+obj.city +" "+obj.name +//// } +//// }else{ +//// println obj.city +" 没有此城市边框信息" +//// } +//// +//// }else{ +//// println d['D']+" 没找到该小区" +//// } +//// +//// } +////},0); +// +// +// +////def city = "杭州" +////def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +////def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) +////def ranges = [] +////for(def d : dbcursor){ +//// ranges << d +////} +////def index =0 +////def updateIndex = 0 +////def delIndex = 0 +////for(def dist : dists){ +////// if(dist.region=="江干"){ +//// def lat = dist.lat/10000000000.0 +//// def lng = dist.lng/10000000000.0 +//// +//// def regionStr =null +//// def region = null +//// for(def r : ranges){ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// regionStr = r.region +//// region = RegionData.formatRegion(dist.city, regionStr) +//// break +//// } +//// } +//// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +//// if(regionStr && region!=dist.region){ +//// +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// delIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// println delIndex+" [del]"+region +" "+dist.name +//// }else{ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name +//// } +//// }else{ +//// index++ +//// println index+" [没找到]"+dist.name +//// } +////// } +////} +// +// +// +// +// +// +//def isInArea(lng, lat, points){ +// List pointList = InsidePolygon.createPoint2(points); +// //Point point = new Point(lng, lat); +// //def flag = InsidePolygon.aboutZone(point, pointList) +// def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) +// return flag; +//} +// +//def getStreetInfo(def city, def glocation){ +// def township = null +// if(glocation){ +// def jsonObject = GaoDeApiUtils.geocode(glocation) +// if(jsonObject && jsonObject.regeocode) { +// def addressComponent = jsonObject.regeocode.addressComponent +// if(addressComponent && addressComponent.city.contains(city)){ +// township = addressComponent.township +// } +// } +// } +// return township; +//} +// +// +////def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////def oracleMap = [:] +////for(def d : oracleDates){ +//// oracleMap.put(d['A'], d['B']) +////} +//// +////def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) +////def chMap = [:] +////for(def h : chDates){ +//// chMap.put(h['A'].toString().replace(".0",""), h['B']) +////} +// +////def citys = CcbCommonUtils.citys; +////def map = [:] +////def rets = [] +////for(def c : citys){ +//// def citycode = cityCodeDataMap.get(c) +//// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") +// +////def weekend = DateUtils.getFirstDayOfWeek(new Date()) +////def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); +////def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) +////def rets = [] +//// +////for(def dist : districts){ +//// def row = [] +//// row << dist.get("_id").toString() +//// row << dist.city +//// row << dist.region +//// row << dist.name +//// if(dist.lng){ +//// row << dist.lng/10000000000.0 +//// row << dist.lat/10000000000.0 +//// }else{ +//// row << "" +//// row << "" +//// } +//// +//// row << statMap.get(dist.region +"##"+dist.name) +//// rets << row +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") +// +////def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +////def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) +//// +////def keys = [] +////for(def d : datas2){ +//// keys << d['A']+"##"+d['C']+"##"+d['B'] +////} +//// +////def index = 0 +////for(def d : datas1){ +//// def k = d['A']+"##"+d['C']+"##"+d['B'] +//// if(!keys.contains(k)){ +//// index ++ +//// println index +" "+k +//// } +////} +// +// +// +// +//def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ +// def q = new BasicDBObject() +// q.put("city", city) +// q.put("weekend", weekend) +// def dbcursor = DIST_SELL_STAT.find(q) +// def map = new ConcurrentHashMap() +// for(def d : dbcursor){ +// map.put(d.region+"##"+d.district_name, d.avg_price) +// } +// return map //} +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx" +//def datas = ExcelUtils.read(fileName, 0, 1) +//def keys = [] +//for(def d : datas){ +// def key = d['A']+"##"+d['B']+"##"+d['C'] +// keys << key +//} +// +//def fileName1 = "D:\\数据\\建行\\房产估值\\指导价\\moban_ccb_0922.xlsx" +//def datas1 = ExcelUtils.read(fileName1, 0, 1) +//def keys1 = [] +//for(def d : datas1){ +// def key = d['A']+"##"+d['B']+"##"+d['C'] +// keys1 << key +//} +// +//for(def k : keys1){ +// if(!keys.contains(k)){ +// println k +// } +//} -def isInArea(lng, lat, points){ - List pointList = InsidePolygon.createPoint2(points); - //Point point = new Point(lng, lat); - //def flag = InsidePolygon.aboutZone(point, pointList) - def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) - return flag; -} +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -def getStreetInfo(def city, def glocation){ - def township = null - if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) - if(jsonObject && jsonObject.regeocode) { - def addressComponent = jsonObject.regeocode.addressComponent - if(addressComponent && addressComponent.city.contains(city)){ - township = addressComponent.township +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ +// def region = RegionData.formatRegion(d['A'], d['B']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") +// if(dist){ +// d['E'] = dist.region +// d['F'] = dist._id.toString() +// } +// } + if(!d['F']){ + def dist = CommonDatautils.findDistrictById(d['A']) +// + if(dist){ +// d['H'] = dist.region +// d['G'] = dist.name + d['F'] = dist.status } } + } - return township; -} +},0) -//def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//def oracleMap = [:] -//for(def d : oracleDates){ -// oracleMap.put(d['A'], d['B']) -//} -// -//def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) -//def chMap = [:] -//for(def h : chDates){ -// chMap.put(h['A'].toString().replace(".0",""), h['B']) -//} -//def citys = CcbCommonUtils.citys; -//def map = [:] -//def rets = [] -//for(def c : citys){ -// def citycode = cityCodeDataMap.get(c) -// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") - -//def weekend = DateUtils.getFirstDayOfWeek(new Date()) -//def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); -//def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) -//def rets = [] -// -//for(def dist : districts){ -// def row = [] -// row << dist.get("_id").toString() -// row << dist.city -// row << dist.region -// row << dist.name -// if(dist.lng){ -// row << dist.lng/10000000000.0 -// row << dist.lat/10000000000.0 -// }else{ -// row << "" -// row << "" +//ExcelUtils.modify(fileName, {d-> +// if(d['A']){ +// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" +// println sql +// sqlInstance236.executeUpdate(sql) // } -// -// row << statMap.get(dist.region +"##"+dist.name) -// rets << row -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") - -//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) -//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) -// -//def keys = [] -//for(def d : datas2){ -// keys << d['A']+"##"+d['C']+"##"+d['B'] -//} -// -//def index = 0 -//for(def d : datas1){ -// def k = d['A']+"##"+d['C']+"##"+d['B'] -// if(!keys.contains(k)){ -// index ++ -// println index +" "+k -// } -//} - - - - -def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ - def q = new BasicDBObject() - q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) - def map = new ConcurrentHashMap() - for(def d : dbcursor){ - map.put(d.region+"##"+d.district_name, d.avg_price) - } - return map -} +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index c4a27815e..6b1009e6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202206 - def endDate = 202208 - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202107 + def endDate = 202109 +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 4a38d4ca3..c2f488b5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -73,15 +73,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_92; def rets = [] - rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] - def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] + rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-05-01" - def date = "2022-06-01" - def endDate = "2022-07-01" - def newDate = "2022-08-01" + def startDate = "2022-07-01" + def date = "2022-08-01" + def endDate = "2022-09-01" + def newDate = "2022-10-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy new file mode 100644 index 000000000..19bf6561b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AddressCut extends BaseBatch{ + static main(args) { + runBatch(new AddressCut(), args); + } + + @Override + public void process(Object args) { + sameAddr() + } + + def sameAddr() { + def citys = CommonDatautils.findAllCity() + def dataMap = [:] + def index = 0 + for (def c : citys) { + index++ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "address"]) + + def addrMap = [:] + for (def dist : dists) { + if (dist.address) { + def rets = addrMap.get(dist.address) + if (!rets) { + rets = [] + } + rets << dist + addrMap.put(dist.address, rets) + } + } + dataMap.put(c, addrMap) + println index +" "+c + } + + + def sameAddrList = [] + for(def c : citys) { + def cityMap = dataMap.get(c) + if(cityMap){ + cityMap.each{addr,rets-> + if(rets && rets.size()>1){ + def keys = new HashSet() + for(def d : rets){ + if(d.name){ + def name = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("七期","").replace("八期","").replace("九期","").replace("十期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("六区","").replace("七区","").replace("八区","").replace("九区","").replace("十区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","") + .replace("十一街区","").replace("十二街区","").replace("十三街区","").replace("十四街区","").replace("十五街区","").replace("十六街区","").replace("十七街区","").replace("十八街区","").replace("十九街区","").replace("一街区","").replace("二街区","").replace("三街区","").replace("四街区","").replace("五街区","").replace("六街区","").replace("七街区","").replace("八街区","").replace("九街区","").replace("十街区","") + .replace("公寓","").replace("别墅","").replace("商住楼","").replace("花园","").replace("南区","").replace("北区","").replace("东区","").replace("西区","") + keys << d.region +"##"+ name + } + } + if(keys && keys.size()>1){ + for(def d : rets) { + if (d.name) { + sameAddrList << [d.city, d.region, d.name, d.address] + } + } + } + } + } + } + } + //ExcelUtils.write("D:\\数据\\小区\\同地址小区.xlsx", sameAddrList, "相同地址小区") + FileUtils.batchCreateFile(sameAddrList, "D:\\数据\\小区\\同地址小区.txt", null, "utf-8", false); + println "完成" + } + + + def addrCut(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + + def fileName = "/home/goojia/gj/address_split.txt" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\address_split.txt" + } + for(def c : citys) { + index++ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"]) + def rets = cut(dists) + FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true); + println index +" "+ c +" "+ rets.size() + } + } + + def cut(rets){ + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + final CountDownLatch latch = new CountDownLatch(rets.size()) + def totalCount = new AtomicLong() + def rows = new CopyOnWriteArrayList() + for(def i=0;i0){ +// dist.commalias = obj.alias.get(0) +// }else{ +// dist.commalias = "" +// } +// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) +// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) +// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); +// dist.houseuse = "11" +// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) +// dist.compid = new ObjectId() //物业公司id +// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) // } // } - - def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) - - for(def dist : districts){ - def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) - if(obj){ - if(obj.alias && obj.alias.size()>0){ - dist.commalias = obj.alias.get(0) - }else{ - dist.commalias = "" - } - dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) - dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) - dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); - dist.houseuse = "11" - dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) - dist.compid = new ObjectId() //物业公司id - dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) - } - } - println index +" "+city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 4.小区建筑年代表" - addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 5.小区费用表" - addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 6.小区房屋用途表" - addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) - println index +" "+ city + " 7.地理位置表" - -// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 - - } +// println index +" "+city +" 小区加载完成 "+districts.size() +// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 1.小区表完成" +// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 2.小区建筑结构表" +// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 3.小区房屋类型表" +// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 4.小区建筑年代表" +// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 5.小区费用表" +// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 6.小区房屋用途表" +// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) +// println index +" "+ city + " 7.地理位置表" +// +//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 +// +// } } def addBigdataCommunityGJRH(def districts, def cityid){ @@ -557,14 +557,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch { def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){ def rets = [] + def num = 0 for(def d : districts) { + num = num + 1 def ret = [] + def detailMap = citySeqnoMap.get(d.cityid) def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap); ret << d.cityid ret << "5" ret << d.commid ret << "1" - ret << seqno + if(detailMap && seqno){ + ret << seqno + }else{ + ret << num + } ret << d.lng ret << d.lat ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index e50059409..386f96f4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -22,25 +22,29 @@ class ExportDistrictRongHe extends BaseBatch{ runBatch(new ExportDistrictRongHe(), args); } + def citys = [] + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + @Override void process(Object args) { + citys = CcbCommonUtils.getRunningCitys(args) def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] - //按天导(每天一个文件夹) + //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH() - exportBigdataBuildingFJW() - exportBigdataCommfeeGj() + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - exportBigdataCommbuildstruGJ() - exportBigdataCommhousetypeGJ() - exportBigdataCommbuildyearGj() - exportBigdataCommhouseuseGJ() - exportBigdataPositionGJ() - exportBigdataDistrictFJW() - exportBigdataAliasFJW() + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// exportBigdataBuildprojFjw() exportBigdataCommlayoutpicFjw() @@ -59,14 +63,14 @@ class ExportDistrictRongHe extends BaseBatch{ // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // // -//// //检查ok文件是否上传 +//// //妫�鏌k鏂囦欢鏄惁涓婁紶 //// exportAllBigdataDealFJW() //// for(def city : citys) { //// def cityCode = cityCodeDataMap.get(city) //// -// //按天导(一个文件夹下) +// //鎸夊ぉ瀵�(涓�涓枃浠跺す涓�) // //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// //按天导(每天一个文件夹) +// //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� // exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) // //// exportBigdataCommunityGJRH(cityCode) @@ -94,261 +98,308 @@ class ExportDistrictRongHe extends BaseBatch{ // exportOk() // } - def exportBigdataCommunityGJRH(cityCode){ - def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMMNAME - data << r.COMMALIAS - data << r.DISTID - data << r.STREET - data << r.STREETNO - data << r.ADDRESS - data << r.LANDAREA - data << r.BUILDAREA - data << r.COMMTYPE - data << r.IFNEW - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << '' - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + def exportBigdataCommunityGJRH(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" + def index = 0 + for(def city : citys){ + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMMNAME + data << r.COMMALIAS + data << r.DISTID + data << r.STREET + data << r.STREETNO + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.COMMTYPE + data << r.IFNEW + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << '' + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + + } + println "BIGDATA_COMMUNITY_FJW 鏁版嵁鏂囦欢杈撳嚭瀹屾垚......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" + println "BIGDATA_COMMUNITY_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommbuildstruGJ(){ - def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDSTRU - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" + println "BIGDATA_COMMBUILDSTRU_FJW 涓婁紶Ftp瀹屾垚......" } - def exportBigdataCommhousetypeGJ(cityCode){ - def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' + def exportBigdataCommhousetypeGJ(){ + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" + println "BIGDATA_COMMHOUSETYPE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommbuildyearGj(){ - def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDYEAR - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" + println "BIGDATA_COMMBUILDYEAR_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommfeeGj(){ - def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMPID - data << r.FEE - data << r.HOUSEUSE - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.FEETYPE - data << r.NOTE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" + println "BIGDATA_COMMFEE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommhouseuseGJ(){ - def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSEUSE - data << r.HOUSECNT - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" + println "BIGDATA_COMMHOUSEUSE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataPositionGJ(){ - def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.OBJECTTYPE - data << r.OBJECTID - data << r.POSITIONTYPE - data << r.SEQNO - data << r.LONGITUDE - data << r.LATITUDE - data << r.ALTITUDE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_POSITION_FJW 上传Ftp完成......" + println "BIGDATA_POSITION_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataDistrictFJW(){ - def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" + println "BIGDATA_DISTRICT_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataAliasFJW(){ - def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.city_id - data << r.ca_id - data << r.address_id - data << r.address_name - data << r.status - data << r.exp_dt - data << r.version - data << r.is_key_name - data << r.data_source - data << r.description - data << r.alias_grade - data << r.addr_type_cd - data << r.ifexamine - data << r.ifdisplay - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "ADDRESS_ALIAS_FJW 上传Ftp完成......" + println "ADDRESS_ALIAS_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataDealFJW(dealdate){ @@ -368,7 +419,7 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -453,7 +504,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" + println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr } @@ -468,31 +519,36 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] - if(curDate.toString().endsWith("28")){ - if(new_dists){ - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + - "where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + - " distid not in ${newstr} ";//CITYID='${cityCode}' and - def rets1 = createDeal(sql1) - def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + - " where distid in ${newstr}" - def rets2 = createDeal(sql2) - allRows.addAll(rets1) - allRows.addAll(rets2) - }else{ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and - allRows = createDeal(sql) + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("28")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + "where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + + " distid not in ${newstr} ";//CITYID='${cityCode}' and + def rets1 = createDeal(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + " where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = createDeal(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";// + allRows = createDeal(sql) + } } } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${endDateStr}" } def createDeal(sql){ @@ -653,54 +709,68 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_DEAL_FJW 导出完成......" + println "BIGDATA_DEAL_FJW 瀵煎嚭瀹屾垚......" } - def exportBigdataBuildingFJW(cityCode){ - def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.BUILDALIAS - data << r.DISTID - data << r.COMMID - data << r.BUILDYEAR - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.FLOORCNT - data << r.UNITCNT - data << r.HOUSECNT - data << r.IFELEVATOR - data << r.ELEVATORCNT - data << r.ELEVATORHOUSE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.VALID_FLAG - allRows << data - }); + def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + def index = 0 + for(def city : citys) { + + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.BUILDALIAS + data << r.DISTID + data << r.COMMID + data << r.BUILDYEAR + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.FLOORCNT + data << r.UNITCNT + data << r.HOUSECNT + data << r.IFELEVATOR + data << r.ELEVATORCNT + data << r.ELEVATORHOUSE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.VALID_FLAG + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + println "BIGDATA_BUILDING_FJW 涓婁紶Ftp瀹屾垚......" } @@ -724,40 +794,40 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "EVALUATE_RATE_FJW 上传Ftp完成......" + println "EVALUATE_RATE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" + println "BIGDATA_BUILDPROJ_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" + println "BIGDATA_COMMLAYOUTPIC_FJW 涓婁紶Ftp瀹屾垚......" } def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + dataCodeFileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" } FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy new file mode 100644 index 000000000..0e94c9551 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session + +import com.jcraft.jsch.*; + + +class SSHHelperTest { + + private Session session = null; + private Channel channel = null; + + private String sftpHost = "192.168.0.86"; + private int sftpPort = 17495; + private String sftpUserName = "ccb"; + private String sftpPassword = "ccb!@#"; + private int timeout = 30000; + /** + * 获取连接 + * @return + */ + private ChannelExec getChannelExec() { + try { + if (channel != null && channel.isConnected()) { + return (ChannelExec) channel; + } + JSch jSch = new JSch(); + if (session == null || !session.isConnected()) { + session = jSch.getSession(sftpUserName, sftpHost, sftpPort); + session.setPassword(sftpPassword); + Properties config = new Properties(); + config.put("StrictHostKeyChecking", "no"); + session.setConfig(config); + session.setTimeout(timeout); + session.connect(); + } + channel = session.openChannel("exec"); + } catch (Exception e) { + if (session != null) { + session.disconnect(); + session = null; + } + channel = null; + } + return channel == null ? null : (ChannelExec) channel; + } + + /** + * 关闭连接 + */ + private void closeChannel() { + try { + if (channel != null) { + channel.disconnect(); + channel = null; + } + if (session != null) { + session.disconnect(); + session = null; + } + } catch (Exception e) { + System.out.println(e); + } + } + + /** + * 执行服务器端命令 + */ + public boolean executeCommand(String command) { + boolean flag = false; + ChannelExec channelExec = getChannelExec(); + if (channelExec == null) { + return false; + } + try { + channelExec.setInputStream(null); + channelExec.setErrStream(System.err); + channelExec.setCommand(command); + + InputStream ins = channelExec.getInputStream(); + channelExec.connect(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8")); + String line = null; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + reader.close(); + closeChannel(); + + flag = true; + } catch (Exception e) { + System.out.println(e); + flag = false; + } + return flag; + } + + public static void main(String[] args) { + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110")); + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 33b350c5c..3be6ffa9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -473,6 +473,7 @@ class CommonDatautils { if(dbo.mapping_alias){ detailMap.mapping_alias = dbo.mapping_alias; } + detailMap.estate_type = dbo.estate_type2 } }catch(Exception e){ @@ -754,7 +755,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","森望芳草绿岸") +// q.put("name","湖畔佳苑") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index dfd414504..ea6d2717b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -11,6 +11,10 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern @@ -26,7 +30,9 @@ class DistrictInfoExportBatch extends BaseBatch{ public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl) + public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { runBatch(new DistrictInfoExportBatch(), args) @@ -87,7 +93,12 @@ class DistrictInfoExportBatch extends BaseBatch{ def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); - for(def rMap : districts){ + + //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 + saveOrUpdateFzDist(districts) + def fzDistricts = getFzDistricts(city) + + for(def rMap : fzDistricts){ //价格没有也提供 // def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) // if(dbo){ @@ -139,6 +150,51 @@ class DistrictInfoExportBatch extends BaseBatch{ return result } + def saveOrUpdateFzDist(districts){ + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;i + udate.put(k,v) + } + udate.removeField("_id") + udate.put("m_date", new Date()) + FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate)); + }else{ + FZ_DIST_COLL.save(d) + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + + } + + def getFzDistricts(def city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = [] + FZ_DIST_COLL.find(q).each{it-> + rets << it + } + return rets + } + def processAlias(alias, def addr_alias){ def newAlias = [] if(alias){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 276c86cec..45f3cbe35 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{ def charset = "GBK" public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } @@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{ return sb.toString() } + def getFzDistricts(def city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = [] + FZ_DIST_COLL.find(q).each{it-> + rets << it + } + return rets + } + def createDatas(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", null); - def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - districts.addAll(removeDists) +// def districts = CommonDatautils.findAllDistricts(city, "district", null); +// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); +// districts.addAll(removeDists) + def districts = getFzDistricts(city) def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{ curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } + }else{ + curPrice = specialDistsPriceMap.get(rMap.name) + if(curPrice){ + def randMarkup = nextDouble(0, 1) + curPrice = (int)(curPrice*randMarkup/100 + curPrice) + } } def m = [] - if(curPrice){ + if(curPrice && dbo){ m << dbo.get("_id").toString()//均价ID }else{ m << new ObjectId().toString() @@ -135,7 +154,11 @@ class DistrictPriceExportBatch extends BaseBatch{ m << rMap.city m << rMap.region m << rMap.name - m << curPrice + if(curPrice){ + m << curPrice + }else{ + m << 0 + } m << curDate.toInteger(); result << m } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index a624061d9..78537bc39 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -98,9 +98,11 @@ class FzDataResultStat { def dist = CommonDatautils.findDistrictById(list[1]) def flag = "有效" - if(dist.status!=0){ + + if(dist && dist.status!=0){ flag = "失效" } + rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 7cdcc589a..43203f42f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy index 51a06b0c3..deff62712 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy @@ -18,9 +18,10 @@ class RegionUpdate { } def run1(){ - def districts = CommonDatautils.findAllFiledDistrict("district","漳州"); + def city = "四平" + def districts = CommonDatautils.findAllFiledDistrict("district",city); - def regions = CommonDatautils.getRegionListByCity("漳州") + def regions = CommonDatautils.getRegionListByCity(city) def index = 0 def addIndex = 0 DistrictRegionTransfer transfer = new DistrictRegionTransfer() @@ -28,6 +29,9 @@ class RegionUpdate { if(districts){ //根据经纬度获取经纬度所在区域 for(def dist : districts){ + if(dist.region!="公主岭"){ + continue + } index++ if(!regions.contains(dist.region)){ @@ -35,28 +39,28 @@ class RegionUpdate { println regionErrorIndex +" "+dist.name } -// def region = getRegionByLocation(dist.lng/10000000000d+","+dist.lat/10000000000d); -// if(!region){ -// def detailMap = getRegionByAddress("漳州", dist.address); -// if(detailMap && detailMap.region && detailMap.region != dist.region){ -// addIndex++ -// dist.lng = (detailMap.lng.toDouble()*10000000000).longValue() -// dist.lat = (detailMap.lat.toDouble()*10000000000).longValue() -// -// def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble()) -// dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue() -// dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue() -// -// println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name -// transfer.transfer(dist, dist.city, detailMap.region) -// } -// }else{ -// if(region && region!=dist.region){ -// addIndex++ -// println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name -// transfer.transfer(dist, dist.city, region) -// } -// } + def region = getRegionByLocation(city, dist.lng/10000000000d+","+dist.lat/10000000000d); + if(!region){ + def detailMap = getRegionByAddress(city, dist.address); + if(detailMap && detailMap.region && detailMap.region != dist.region){ + addIndex++ + dist.lng = (detailMap.lng.toDouble()*10000000000).longValue() + dist.lat = (detailMap.lat.toDouble()*10000000000).longValue() + + def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble()) + dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue() + dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue() + + println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name + transfer.transfer(dist, dist.city, detailMap.region) + } + }else{ + if(region && region!=dist.region){ + addIndex++ + println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name + transfer.transfer(dist, dist.city, region) + } + } } } } @@ -85,14 +89,16 @@ class RegionUpdate { return detail } - def getRegionByLocation(def location){ + def getRegionByLocation(city, def location){ def json = GaoDeApiUtils.geocode(location); - def city = "漳州" def region = null; if (json && json.info=="OK") { def regeocode = json.regeocode if(regeocode && regeocode.addressComponent){ region = regeocode.addressComponent.district + if(!region){ + region = regeocode.addressComponent.township + } } } if(region){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index a13780761..6ccaabde3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,13 +48,13 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-07-01" + static def statDate = "2022-09-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es - task.esfhouseToSaveEs();//二手房保存到es +// task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } @@ -284,8 +284,8 @@ class YouDaDataParser { } - def esfParser(){ - def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"] + def esfParser(path){ + def files = [path] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 418b6c102..cb615604a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -29,7 +29,30 @@ class DataMatch { // task.run3() // println task.findAddress("昌州街道棠香路") - task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx") + } + + def fzWeiyuanshuMatch(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> +// if(!d['G']){ +// def arr = OpenApiClient.search("福州", "连江", d['C']) +// if(arr){ +// d['E'] = arr.city +// d['F'] = arr.district +// d['G'] = arr.name +// } +// } + + def results = null + if(d['H'] && d['H']=="F" && d['C']){ + results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "连江", null) + } + if(results && results.precise>=0){ + d['J'] = results.district + d['K'] = results.name + } + },0) } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 9f3670a44..ae1dee2e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -67,6 +67,7 @@ class FangjiaClientUtils { + //分词 省,市,区 public static def addressStand(def city, def address){ def jsonObj = null def content = null @@ -84,6 +85,22 @@ class FangjiaClientUtils { return jsonObj; } + //分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼 + public static def addressStructure(def city, def address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static def arroundPrice(city,address){ def jsonObj = null def content = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy index 2e9a536ef..fe2ab3332 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy @@ -11,9 +11,10 @@ import org.apache.lucene.queryparser.classic.QueryParser class DongGuanBankTest { static main(args) { def task = new DongGuanBankTest() - def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx" + def path = "D:\\数据\\东莞银行\\东莞银行测试_输出单价(2).xlsx" // task.run(path) - task.addPrice(path) +// task.addPrice(path) + task.addMinMaxPrice(path) } def run(path){ @@ -92,7 +93,7 @@ class DongGuanBankTest { def set = null try{ def aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')" + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210901 and region='${region}' and district_name in ('${aliasStr}') and root='anjuke.com'" set = new HashSet() ElasticUtils.executeSqlQuery(sql,{data-> @@ -132,4 +133,26 @@ class DongGuanBankTest { return avg_price } + def addMinMaxPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['V']){ + def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district") + if(dist){ + def alias = [] + if(!dist.alias){ + alias = [] + }else{ + alias = dist.alias + } + alias << dist.name + def map = findMaxAvgPrice(dist.city, dist.region, alias) + if(map){ + d['V'] = map.max + d['W'] = map.min + } + } + } + }) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index be10cf533..3d79e39fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,9 +19,44 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() - task.houseRun() +// task.houseRun() // task.statHouseUpdateCount() + + task.statDistHouse() + } + + def statDistHouse(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\大连有效小区确认_2256--查好-1.xlsx",{d-> + if(d['B']){ + def alias = [] + if(d['G']){ + def s = d['G'].replace("[", "").replace("]", "").replace("'","").replace('"',"") + def arrs = null + if(s){ + arrs = s.split(",") + } + if(arrs){ + arrs = arrs as List + if(!arrs.contains(d['D'])){ + arrs << d['D'] + } + alias = arrs + }else{ + alias << d['D'] + } + }else{ + alias << d['D'] + } + + def distCount = findHouseCount(d['B'], d['C'], alias, 20190101, 20221001) + if(distCount){ + d['E'] = distCount + } + println d['B']+" "+d['C']+" "+d['D']+" "+distCount + } + + }) } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy new file mode 100644 index 000000000..444fddbfc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +EsDataSource dataSource = new EsDataSource(); +String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + +findLogs() + +def findLogs(){ + def total = 0 + try{ + +// String sql = "select * from fangjia-log* where request like '/jnBank%'" +// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + def q = QueryBuilders.fuzzyQuery("request", "/jnBank") + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder); + println houses.size() + def set = new HashSet() + if(houses) { + for (def d : houses) { + def request = d.request + if(request){ + def arrs = request.split("&") + if(arrs){ + for(def a : arrs){ + if(a.contains("CITY_CD")){ + def list = a.split("=") + if(list && list.size()>=2){ + set.add(list[1]) + } + } + } + } + } + } + } + + println set + + +// ElasticUtils.executeSqlQuery(sql,{ data-> +// println data.request +// },"192.168.0.25") + }catch(Exception e){ + e.printStackTrace() + } + + return total +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 2560bb842..eac332041 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 42489aafe..b63201d66 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -49,7 +49,15 @@ public class BeiKeCrawlByChrome { WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) - citys = citys.reverse() +// citys = citys.reverse() +// def index = 0 +// for(def c : citys){ +// index++ +// if(c.city=="合肥"){ +// break +// } +// } +// citys = citys[index..citys.size()-1] def start = 0 // for(def i=0;i=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) @@ -469,11 +470,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["石家庄","贵阳"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 2b8b3cf9b..cfc3ad6d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.rabbitmq.client.AMQP import com.rabbitmq.client.impl.AMQImpl @@ -35,11 +36,12 @@ class PaimaiDataSend { static main(args) { def task = new PaimaiDataSend() // task.jdRunByMongo() -// task.jdRun() + // task.aliZcRun() - task.aliZcRun2() -// task.aliFpRun() +// task.aliZcRun2() + task.aliFpRun() // task.rmfysszcRun() +// task.jdRun() // task.checkUrl() // task.paimaiSendToEs() @@ -66,7 +68,8 @@ class PaimaiDataSend { def rmfysszcRun(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) + q.put("province","江苏省") // q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ @@ -149,10 +152,12 @@ class PaimaiDataSend { def jdRun(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) + q.put("province","江苏") // q.put("parser_status", new BasicDBObject('$ne',3)) // q.put("url","https://paimai.jd.com/115669796") def dbcursor = jd_coll.find(q).batchSize(3000) + def rets = [] for(def d : dbcursor){ index++ println "jd:"+index @@ -175,23 +180,57 @@ class PaimaiDataSend { result.put("notice_detail_content", d.notice_detail_content.content) json.put("result", result) - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) + rets << json + if(rets.size()==10000){ + println "共:"+rets.size() + sendJD(rets) + rets.clear() + } + } - def udata = new BasicDBObject() - udata.put("parser_status", 3) - udata.put("m_date", new Date()) - jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + if(rets.size()>0){ + println "共:"+rets.size() + sendJD(rets) } } + def sendJD(def rets){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i + def m = [:] + m.region = r.DISTNAME + m.cityId = r.CITYID + m.distId = r.DISTID + list << m + }) + return list; + } + + def run(){ def rets = findAllCity() for(def city : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy new file mode 100644 index 000000000..5312d274a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy @@ -0,0 +1,94 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.core.RegionData +import groovy.sql.Sql + +def newRegionCodeFile = "D:\\数据\\建行\\二期\\行政区处理_20220923.xlsx" + +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def specialCityMap = ["内蒙古##兴安":"兴安盟","四川##甘孜":"甘孜州","云南##文山":"文山州","青海##玉树":"玉树州"] + +//ExcelUtils.modify(newRegionCodeFile,{d-> +// if(d['A'] && !d['I']){ +// def city = specialCityMap.get(d["A"]+"##"+d["B"]) +// if(!city){ +// city = RegionData.formatCity(d['B']) +// } +// if(city){ +// def region = RegionData.formatRegion(city, d['E']) +// if(region){ +// d['I'] = city +// d['J'] = region +// } +// } +// } +//}) + +def newDatas = ExcelUtils.read(newRegionCodeFile,0,1) +def newDataMap = [:] +for(def d : newDatas){ + def key = d['I']+"##"+d['J'] + def rets = newDataMap.get(key) + if(!rets){ + newDataMap.put(key, d) + } +} + +def ccbMap = getCcb92City(sqlInstance) +ExcelUtils.modify("D:\\数据\\亲邻\\region_code.xlsx",{d-> + if(!d['C'] && d['D']){ + def dbo = CommonDatautils.findAlCityDetail(d['A']) + if(dbo){ + d['C'] = dbo.province + } + } +// if(!d['D']){ +// def m = newDataMap.get(d['A']+"##"+d['B']) +// if(m){ +// d['C'] = m['A'] +// d['D'] = m['B'] +// d['E'] = m['C'] +// d['F'] = m['D'] +// d['G'] = m['E'] +// } +// def m = ccbMap.get(d['A']+"##"+d['B']) +// if(m){ +// d['D'] = m.city +// d['E'] = m.cityId +// d['F'] = m.distId +// d['G'] = m.region +// } +// } +}) + +def getCcb92City(sqlInstance){ + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj" + def map = [:] + sqlInstance.eachRow(sql, { r -> + def city = getCityName(r.CITYID) + def region = RegionData.formatRegion(city, r.DISTNAME) + def m = [:] + m.city = city + m.region = r.DISTNAME + m.cityId = r.CITYID + m.distId = r.DISTID + map.put(city+"##"+region, m) + }) + return map; +} + +def getCityName(cityId){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def city = null + cityCodeDataMap.each{cityName, id-> + if(id == cityId){ + city = cityName + } + } + return city; +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy index b92188bdd..e81f4cb67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy @@ -25,7 +25,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) def q = new BasicDBObject() // q.put("province",new BasicDBObject('$in', ["江苏","江苏省"])) q.put("status",new BasicDBObject('$ne', 1)) - def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000) + def dbcursor = school_coll.find(q).batchSize(5000) def rets = [] for(def d : dbcursor){ JSON json = new JSONObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 926581423..24eeb3c76 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -647,9 +647,9 @@ class CZDataBatch extends BaseBatch{ } } - if(!priceMap){ - priceMap = findRegionRentPrice(priceMap, city, region, estate_type2) - } +// if(!priceMap){ +// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2) +// } //5室及以上价格小于2000去除 if(priceMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 70082d882..a50827e27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -209,4 +209,20 @@ class SFTP { return isDirExistFlag; } + public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) { + long fileSize;//文件大于等于0则存在 + try { + SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath); + fileSize = sftpATTRS.getSize(); + } catch (Exception e) { + e.printStackTrace() + fileSize = -1;//获取文件大小异常 + if (e.getMessage().toLowerCase().equals("no such file")) { + fileSize = -2;//文件不存在 + } + } + return fileSize; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy index d114be591..a0c3ec24f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -5,6 +5,9 @@ import com.fangjia.script.temp.shenjian.util.EsUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.ElasticUtils +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + /** * 补充学校经纬度 * @author shenjian @@ -12,16 +15,25 @@ import com.fangjia.script.util.ElasticUtils */ class FillLngLat { + static ExecutorService executorService = Executors.newFixedThreadPool(4) static void main(String[] args) { - ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { data -> - println(data) - def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) - if (lngLat != null && lngLat.size() == 2 ) { - JSONObject jsonObject = new JSONObject() - jsonObject.put("lon", Double.valueOf(lngLat['lng'])) - jsonObject.put("lat", Double.valueOf(lngLat['lat'])) - EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) - } + ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + final def data = dataT + executorService.submit(new Runnable() { + @Override + void run() { + println(data) + def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + if (lngLat != null && lngLat.size() == 2 ) { + JSONObject jsonObject = new JSONObject() + jsonObject.put("lon", Double.valueOf(lngLat['lng'])) + jsonObject.put("lat", Double.valueOf(lngLat['lat'])) + EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) + } + } + }) + }) + executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 147d08336..2bc9b09b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -73,11 +73,11 @@ class SeaweedAroundPoi extends BaseBatch { // void run() { def city = obj.city if (obj.lng) { - //def lng = SeaweedUtils.formatLatLng(obj.lng) - //def lat = SeaweedUtils.formatLatLng(obj.lat) + def lng = SeaweedUtils.formatLatLng(obj.lng) + def lat = SeaweedUtils.formatLatLng(obj.lat) // {'lon': 121.810877, 'lat': 31.737642} - def lng = 121.810877 - def lat = 31.737642 +// def lng = 121.810877 +// def lat = 31.737642 List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { String jsonStr = JSONObject.toJSONString(pois) @@ -178,11 +178,11 @@ class SeaweedAroundPoi extends BaseBatch { // put("写字楼", "120201"); // put("学校", "141200|141201|141202|141203|141204"); // put("银行", "1601|1603"); -// put("大型超市", "0604"); + put("大型超市", "0604"); // put("商场", "0601"); - put("餐饮", "0501|0502|0503|0504"); +// put("餐饮", "0501|0502|0503|0504"); // put("旅游景点", "110000"); -// put("公园广场", "110100|110101|110102|110103|110104|110105") + put("公园广场", "110100|110101|110102|110103|110104|110105") // put("图书馆", "140500"); // put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); // put("丧葬设施", "071900|071901|071902|071903") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy new file mode 100644 index 000000000..66abaacf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiFromExcel { + + static void main(String[] args) { + def types = listAllType() + dealPoi(new BasicDBObject(), types) + } + + static dealPoi(DBObject query, def types) { + com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item -> + def city = item['H'] + def lng = item['AQ'] + def lat = item['AR'] + if (lng) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr) + } + Thread.sleep(1000) + print(lng + " " + lat) + } + }); + }; + + + + public static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + }; + + public static final Map KEYWORDS_MAP = new HashMap() { + { +// put("餐饮", "05"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校", "141200|141201|141202|141203|141204"); +// put("银行", "1601|1603"); + put("大型超市", "0604"); +// put("商场", "0601"); +// put("餐饮", "0501|0502|0503|0504"); +// put("旅游景点", "110000"); + put("公园广场", "110100|110101|110102|110103|110104|110105") +// put("图书馆", "140500"); +// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); +// put("丧葬设施", "071900|071901|071902|071903") + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 0aa5d9ce5..17b02bb6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -147,10 +147,12 @@ class GaoDeUtils { params.page = "${page}".toString() page++ params.key = this.getGaodeKey("v3/place/around") + //params.key = "25b2d459fd133e22f8abcebeac972692" params.offset = "100" JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { params.key = this.getGaodeKey("v3/place/around") + //params.key = "25b2d459fd133e22f8abcebeac972692" params.offset = "100" json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 9a9fd90ad..7c1ad2c31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils - +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{ } def rowIndex = 0; + + DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018) + DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018) static main(args) { runBatch(new DataStandardBatch(), args) } @@ -87,9 +91,9 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) - districtMap.putAll(detailMap) + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) } while(flag){ @@ -116,9 +120,9 @@ class DataStandardBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger successIndex = new AtomicInteger(0) datas.each { dbo -> - fixedThreadPool.execute( - new Runnable() { - void run() { +// fixedThreadPool.execute( +// new Runnable() { +// void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" if(dbo.name){ @@ -158,12 +162,12 @@ class DataStandardBatch extends BaseBatch{ }finally { latch.countDown() } - } - } - ); +// } +// } +// ); } - latch.await() +// latch.await() long end = System.currentTimeMillis() println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") @@ -370,39 +374,44 @@ class DataStandardBatch extends BaseBatch{ if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ district = null; }else{ - district = getMatchNameFromMap(data.city, data.region, data.name, data.address) + district = getMatchNameFromMap(data) if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区” - district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address) + def copyObj = data + copyObj.name = data.name+"小区" + district = getMatchNameFromMap(copyObj) } - if(!district){ def newName = filterSpacialChar(data.name);//包含特殊符号 if(newName) { - district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address) + def copyObj = data + copyObj.name = newName.replaceAll("_", "") + district = getMatchNameFromMap(copyObj) if(!district){ def s1 = newName.substring(0, newName.indexOf("_")) if (s1) { //特殊符号前的内容匹配 //district = getMatchName(data.city, data.region, s1, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s1, data.address) + copyObj.name = s1 + district = getMatchNameFromMap(copyObj) if (!district) { - def s2 = newName.replace(s1, "").replaceAll("_", "") def newName2 = null; if (s2 && s2.length() >= 3) { //特殊符号后的内容匹配 // district = getMatchName(data.city, data.region, s2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s2, data.address) + copyObj.name = s2 + district = getMatchNameFromMap(copyObj) if (district) { type = 3; println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]" } } else { newName2 = newName.replaceAll("_", "") + copyObj.name = newName2 //district = getMatchName(data.city, data.region, newName2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, newName2, data.address) + district = getMatchNameFromMap(copyObj) if (district) { type = 4; println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]" @@ -410,15 +419,20 @@ class DataStandardBatch extends BaseBatch{ } if (!district) { - - district = getMatchNameFromMap(data.city, null, newName2, data.address) + copyObj.name = newName2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if(!district){ - district = getMatchNameFromMap(data.city, null, s1, data.address) + copyObj.name = s1 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 5; } else { if (s2 && s2.length() >= 3) { - district = getMatchNameFromMap(data.city, null, s2, data.address) + copyObj.name = s2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 6; } @@ -446,11 +460,14 @@ class DataStandardBatch extends BaseBatch{ }catch (Exception e){ addr = data.address; } - district = getMatchAddressFromMap(data.city, data.region, addr, data.name) + def copyObj = data + copyObj.address = addr + district = getMatchAddressFromMap(copyObj) if(district){ type = 8; }else{ - district = getMatchAddressFromMap(data.city, null, addr, data.name) + copyObj.region = null + district = getMatchAddressFromMap(copyObj) if(district){ type = 9; } @@ -460,25 +477,31 @@ class DataStandardBatch extends BaseBatch{ if(!district){ def districts = getMatchPingYing(data.city, data.region, data.name, addr); - district = checkMatchAddress(districts, addr); + district = checkMatchAddress(districts, addr, data); if(district){ type = 10; }else if(districts.size()==1){ - district = districts[0]; - type = 14 + district = getFilerMatchDistrict(districts[0], data); + if(district){ + type = 14 + } } } if(!district){ - district = getOpenApiMatch(data.city, data.region, data.name, addr); + def copyObj = data + copyObj.address = addr + district = getOpenApiMatch(copyObj) if(!district && addr){ - district = getOpenApiMatch(data.city, data.region, addr, data.name); + district = getOpenApiMatch(copyObj) } if(district){ if(data.region != district.region){ type = 12;//区域不同 district = null; }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name) + district.score = score //此处用来做测试 type=15 } } @@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{ return null; } - def checkMatchAddress(def districts, def address){ + def checkMatchAddress(def districts, def address, def data){ def district = null; if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){ def tempDist = districts[0]; @@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(district){ + return getFilerMatchDistrict(district, data) + } return district; } @@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, String address){ - //查询房价网是否存在该小区 - def dbo = findSameNameFromMap(city, region, disName); + def getMatchNameFromMap(def data){ + //查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price + def dbo = findSameNameFromMap(data.city, data.region, data.name); def addrFlag = null - if(address){ - addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(data.address){ + addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(addrFlag){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } if(!dbo){ //别名 - def distrs = findAliasNameFromMap(city, region, disName); + def distrs = findAliasNameFromMap(data.city, data.region, data.name); if(distrs && distrs.size()==1){ dbo = distrs[0] - }else if(distrs && distrs.size()>1 && address){ + }else if(distrs && distrs.size()>1 && data.address){ for(def d : distrs){ - if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ + if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } //地址 if(!dbo){ - def result = findAddressFromMap(city, region, disName) + def result = findAddressFromMap(data.city, data.region, data.name) if(result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && address){ + }else if(result && result.size()>1 && data.address){ for(def d : result){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } } + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } return dbo; } + def getFilerMatchDistrict(dbo, data){ + def types = [] + if(data.estate_type){ + types = getEstateTypes(data.estate_type) + } + if(types.size()>0 && types.size() == 1){ + if(types.contains(dbo.estate_type)){ + return dbo; + }else{ + def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","物业类型不匹配") + THIRD_MATCH_FILTER_COLL.save(data) + } + return null + } + }else{ + def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger()) + def price = priceMap.get(dbo.estate_type) + if(price && data.avg_price && data.avg_price.toString().isNumber()){ + def pre = (data.avg_price-price)*100/price + if(pre>=-20 && pre<=20){ + return dbo; + }else{ + def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","房源价格过滤") + THIRD_PRICE_FILTER_COLL.save(data) + } + return null + } + } + } + return dbo + } + + + def findHousePrice(def city, def region, def name, def startDate){ + def typePriceMap = [:] + try{ + String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'" + def houseMap = [:] + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.estate_type){ + def key = "" + if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){ + key = "普通住宅" + }else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){ + key = "商住" + }else if( data.estate_type.contains("别墅")){ + key = "别墅" + }else{ + key = "其它" + } + def houses = null + if(key){ + houses = houseMap.get(key) + if(!houses){ + houses = [] + } + } + if(data.avg_price){ + houses << data.avg_price + houseMap.put(key, houses) + } + } + }) + if(houseMap){ + houseMap.each{k,v-> + if(v && v.size()>=3){ + def price = (int)(v.sum()/v.size()) + typePriceMap.put(k, price) + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return typePriceMap + } + + def getEstateTypes(str){ + def types = [] + if(str){ + if(str.contains("住宅") || str.contains("公寓")){ + types << "普通住宅" + } + if(str.contains("商住") || str.contains("商务")){ + types << "商住" + } + if(str.contains("别墅")){ + types << "别墅" + } + } + return types; + } /*def getMatchAddress(def city, def region, def address, def name, def cat){ def dbo = CommonDatautils.findSeaweed(city, region, address, cat); @@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{ } return dbo; }*/ - def getMatchAddressFromMap(def city, def region, def address, def name){ + def getMatchAddressFromMap(def copyObj){ def dbo = null - if(address && isStandardRoad(address)){ - dbo = findSameNameFromMap(city, region, address); + def name = copyObj.name + if(copyObj.address && isStandardRoad(copyObj.address)){ + dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address); } if(!dbo){ def result = null - if(address && isStandardRoad(address)){ - result = findAddressFromMap(city, region, address); + if(copyObj.address && isStandardRoad(copyObj.address)){ + result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address); } if(result && result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && name){ + }else if(result && result.size()>1 && copyObj.name){ def score = null; for(def d : result){ - if(d.name == name){ + if(d.name == copyObj.name){ dbo = d; break; }else{ @@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{ } } if(!dbo){ - result = findAddrAliasFromMap(city, region, address); + result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ @@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(dbo){ + return getFilerMatchDistrict(dbo, copyObj) + } + } return dbo; } @@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{ if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] } } } @@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{ } - def getOpenApiMatch(def city, def region, def name, def address){ - if(StringUtils.isNotEmpty(name)){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false) + def getOpenApiMatch(def data){ + if(StringUtils.isNotEmpty(data.name)){ + def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) if(jsonObject && jsonObject.credit>0 ){ - return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit] + def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } + return dbo } } return null; @@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) +// qdata.put("city","北京") +// qdata.put("source","安居客") +// qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -925,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{ udata2.put("match_region", district.region); udata2.put("match_name", district.name); udata2.put("match_address", district.address); + if(district.score){ + udata2.put("score", district.score); + } udata2.put("avg_price", data.avg_price); udata2.put("type", type); udata2.put("tid", data._id.toString()); @@ -948,9 +1091,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } @@ -962,9 +1105,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -990,18 +1133,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1016,18 +1159,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1052,9 +1195,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1068,9 +1211,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1093,9 +1236,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1106,9 +1249,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 1ae32a425..e76898c7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "我爱我家"// - def citys = findCrawlCity(source, 202207); + def source = "安居客"// + def citys = findCrawlCity(source, 202208); for(def city : citys){ - def count1 = findCount(source, city, 202207); - def count2 = findCount(source, city, 202208); + def count1 = findCount(source, city, 202208); + def count2 = findCount(source, city, 202209); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 886acc0b1..f954743e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils @@ -11,7 +14,8 @@ import org.bson.types.ObjectId * 福州农商行 * 将抓取的小区均价同步到第三方均价表中 */ -class SynchDistrictPrice { +@BatchOwner(Watcher.GaoJuan) +class SynchDistrictPrice extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); @@ -20,16 +24,22 @@ class SynchDistrictPrice { // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - //"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" - def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"] + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" + def esSources = ["中原地产":"中原地产"] + + static main(args) { - def task = new SynchDistrictPrice() -// task.run() - task.run1() + runBatch(new SynchDistrictPrice(), args); + } + + @Override + public void process(Object args) { + run1() } def run1(){ + println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def citys = CommonUtils.findAllCity() @@ -47,7 +57,7 @@ class SynchDistrictPrice { def names = new HashSet() def rets = [] ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> - def s = d.city +"#"+d.region+"#"+d.name + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type if(!names.contains(s)){ names << s def data = new BasicDBObject() @@ -57,6 +67,7 @@ class SynchDistrictPrice { data.put("address", d.address) data.put("avg_price", d.avg_price) data.put("source", mgSource) + data.put("estate_type",d.property_type) data.put("s_date", SHORT_DATE) data.put("url", d.url) data.put("c_date", new Date()) @@ -134,7 +145,7 @@ class SynchDistrictPrice { println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source) if(obj){ updateIndex++ // thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) @@ -142,6 +153,7 @@ class SynchDistrictPrice { def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) + udata.put("estate_type", r.estate_type) udata.put("m_date",new Date()) thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) @@ -171,13 +183,16 @@ class SynchDistrictPrice { } - def findIsExists(def city, def region, def name, def sDate, def source){ + def findIsExists(def city, def region, def name, estate_type, def sDate, def source){ def q = new BasicDBObject() q.put("source", source) q.put("s_date", sDate) q.put("city", city) q.put("region", region) q.put("name", name) + if(estate_type){ + q.put("estate_type",estate_type) + } def obj = thirdPartyColl.findOne(q) return obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 149149fd0..99355ab2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202207); - def count2 = findTotalCount(source, 202208); + def count1 = findTotalCount(source, 202208); + def count2 = findTotalCount(source, 202209); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3efe71b67..320e65b2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -81,8 +81,8 @@ class ThirdDataToGatherBatch extends BaseBatch{ println "小区添加价格完成" districts = addOtherWebsitePrice(districts); println "小区添加第三方价格完成" - districts = addLevel(districts); - println "小区评级完成" +// districts = addLevel(districts); +// println "小区评级完成" saveOrUpdate(districts); println city + "保存修改完成" toLineDB(city) @@ -101,16 +101,16 @@ class ThirdDataToGatherBatch extends BaseBatch{ def tempPrice = d.get(field+"_price") if(!tempPrice){ d.put(field+"_price", m.avg_price.toInteger()) - if(d.fangjia_price && m.avg_price){ - if(d.fangjia_price){ - try{ - d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger())) - }catch(Exception e){ - println "dddd" - } - - } - } +// if(d.fangjia_price && m.avg_price){ +// if(d.fangjia_price){ +// try{ +// d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger())) +// }catch(Exception e){ +// println "dddd" +// } +// +// } +// } } }else{ @@ -208,8 +208,9 @@ class ThirdDataToGatherBatch extends BaseBatch{ query.put("s_date", SHORT_DATE) DBObject udata = new BasicDBObject() - udata.put("level",d.level); - udata.put("last_price", d.last_price); +// udata.put("level",d.level); +// udata.put("last_price", d.last_price); + udata.put("seaweed_id", d._id.toString()) udata.put("fangjia_price", d.fangjia_price); fieldMap.each{k,v-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 3d4e855f2..7c37cae97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -52,6 +52,22 @@ public class OpenApiClient { return false } + public static def search(def city, def district, def name){ + Map params = new HashMap() + params.put("city", city); + params.put("keyword", name); + params.put("district", district) + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8") + JSONObject jsonObj = JSONObject.fromObject(content) + JSONArray array = jsonObj.get("result") + def matchObj = null + if(array){ + matchObj = array[0] + } + return matchObj + } + public static List searchDistrict(double lng, double lat, int r, String city) { DecimalFormat format = new DecimalFormat("#.0000000000"); try { @@ -348,18 +364,20 @@ public class OpenApiClient { } + //此接口地址标准化,带补全 public static JSONObject standard1Action(String city, String address) { try { Map params = new HashMap(); params.put("city", city); params.put("address", address); params.put("token", TOKEN); - params.put("__refresh_cache","1") +// params.put("__refresh_cache","1") - String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + //192.168.0.100:30115 + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8") println("${city} ${address} 程序调用异常") Thread.sleep(6000) } @@ -369,7 +387,7 @@ public class OpenApiClient { println("${city} ${address}") } } catch (Exception e) { - println("${city} ${address} ${e.getMessage()}") + println("[http error] ${city} ${address} ${e.getMessage()}") e.printStackTrace(); } return null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 96febc0f6..09877e434 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -1,188 +1 @@ package com.fangjia.script.wenyan - -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item-> -// if(item["AB"]){ -// return -// } -// String region = null -// String name = item["F"] -// if(name.endsWith("小区")){ -// name = name.replaceAll("小区\$", "") -// } -// region = RegionData.formatRegion(item["B"], item["D"]) -// def results = [] -// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each { -// if(!results.contains(it)){ -// results.add(it) -// } -// } -// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each { -// if(!results.contains(it)){ -// results.add(it) -// } -// } -// if(results.size() == 1){ -// item["Z"] = results[0]._id.toString() -// item["AA"] = results[0].region -// item["AB"] = results[0].name -// println(item) -// } -//}) - - - - -import com.fangjia.core.ProvinceData -import com.fangjia.core.RegionData -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.DataUtil -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection -import net.sf.json.JSONObject -import org.apache.commons.lang.StringUtils -import org.elasticsearch.index.query.BoolQueryBuilder -import org.elasticsearch.index.query.QueryBuilders - -//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) -//def count = 0 -//List> data = Collections.synchronizedList([]) -//data << ["ID","城市","区域","小区名","经度","纬度","街道"] -// -//CommonUtils.findAllCity().each {def city-> -// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> -// if(StringUtils.isBlank(item.township) && item.lng){ -// count++ -// def lng = item.lng/10000000000d -// def lat = item.lat/10000000000d -// def location = "${lng},${lat}" -// JSONObject resp = GaoDeApiUtils.geocode(location) -// def row = [] -// row << item._id.toString() -// row << item.city -// row << item.region -// row << item.name -// row << lng -// row << lat -// BasicDBObject dbo = new BasicDBObject() -// dbo.id = item._id.toString() -// dbo.city = item.city -// dbo.region = item.region -// dbo.name = item.name -// dbo.lng = lng -// dbo.lat = lat -// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ -// row << resp.regeocode.addressComponent.township -// dbo.township = resp.regeocode.addressComponent.township -// } -// println(row) -// data << row -// SEAWEED_TOWNSHIP.insert(dbo) -// } -// }) -//} -//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") - -//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item -> -// if(item["G"] && item["H"] == null){ -// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"]))) -// } -// println(item) -//}) - -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item-> -// def location = "${item["A"]},${item["B"]}" -// JSONObject resp = GaoDeApiUtils.geocode(location) -// if(resp.regeocode && resp.regeocode.formatted_address){ -// item['G'] = resp.regeocode.formatted_address -// println(item) -// } -//}) - -//def count = 0 -//CommonUtils.findAllCity().each {def city-> -// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> -// if(item.township){ -// count++ -// } -// }, new BasicDBObject("township", 1)) -//} -//println(count) -//def GXD_CCB = [:] -//def CCB_FJW = [:] -//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item-> -// if(item["V"] == "级联"){ -// if(item["Y"] == "国信达"){ -// GXD_CCB[item['J']] = item["E"] -// }else if(item["Y"] == "房价网"){ -// CCB_FJW[item['E']] = item["J"] -// } -// } -//} -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item-> -// if(item["V"] == "级联" && item["Y"] == "国信达"){ -// def ccbId = GXD_CCB.get(item['J'], null) -// def fjwId = CCB_FJW.get(ccbId, null) -// if(fjwId != null){ -// item["AB"] = fjwId -//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId) -//// List results = DbUtils.findAroundDistrict(seaweed) -//// long total = 0 -//// long count = 0 -//// for(BasicDBObject s : results){ -//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2) -//// if(avg){ -//// total += avg.avg_price -//// count += 1 -//// } -//// } -//// if(count > 0){ -//// item['Z'] = (int)(total/count) -//// println(item) -//// } -// } -// } -//},sheetIndex = 1) - -ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\小区字段补充.xlsx",{List item-> - BasicDBObject seaweed = DbUtils.seaweed.findById(item["A"]) - if(StringUtils.isBlank(seaweed.create_year)){ - def names = [] - names.add(seaweed.name) - if(seaweed.alias){ - for(def it in seaweed.alias){ - if(StringUtils.isNotBlank(it) && !names.contains(it)){ - names.add(it) - } - } - } - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery() - boolQueryBuilder.must(QueryBuilders.termQuery("city", seaweed.city)) - boolQueryBuilder.must(QueryBuilders.termQuery("region", seaweed.region)) - boolQueryBuilder.must(QueryBuilders.termsQuery("name", names)); - item["B"] = seaweed.city - item["C"] = seaweed.region - item["D"] = seaweed.name - List result = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", boolQueryBuilder) - def buildYears = [] - result.each { - if(StringUtils.isNotBlank(it.build_year) && !StringUtils.containsAny(it.build_year,"无")){ - String build_year = it.build_year - build_year = build_year.replaceAll("((|\\().*()|\\))", "") - build_year = build_year.replaceAll("[年建]", "") -// property_fee = property_fee.replace("元/㎡·月", "元/平米/月") - buildYears.add(build_year) - } - } - buildYears = buildYears.unique() - if(buildYears.size() > 0){ - DbUtils.seaweed.update(new BasicDBObject("_id", seaweed._id), new BasicDBObject('$set', new BasicDBObject('create_year', buildYears[0]))) - println(item) - println(buildYears) - } - } -}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 2f146d11f..fdcf818c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + def static GAO_DE_AK = "90010d53a16ce9df544acab297645106" /** * 经纬度转地址
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/index/DistrictTriggerServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/index/DistrictTriggerServiceImpl.java index 4cb2c5cf2..8d4022ed3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/index/DistrictTriggerServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/index/DistrictTriggerServiceImpl.java @@ -87,8 +87,6 @@ public class DistrictTriggerServiceImpl implements TriggerService { newObj.put("lng", object.get("lng")); newObj.put("lat2", object.get("lat2")); newObj.put("lng2", object.get("lng2")); - newObj.put("lat3", object.get("lat3")); - newObj.put("lng3", object.get("lng3")); newObj.put("estate_type", object.get("estate_type")); newObj.put("estate_type2", object.get("estate_type2")); newObj.put("build_type", object.get("build_type")); @@ -138,15 +136,6 @@ public class DistrictTriggerServiceImpl implements TriggerService { newObj.put("lat_lng", lat + "," + lng); } } - -// if (null != object.get("lat3") && null != object.get("lng3")) { -// double lat3 = (double)((Long)object.get("lat3") / 10000000000.0); -// double lng3 = (double)((Long)object.get("lng3") / 10000000000.0); -// if (lat3 >= -90 && lat3 <= 90 && lng3 >= -180 && lng3 <= 180) { -// newObj.put("lat_lng3", lat3 + "," + lng3); -// } -// } - if (object.get("kind") != null) { String[] kinds = ((String) object.get("kind")).split(","); if (ArrayUtils.contains(kinds, DistrictKindEnum.bieshu.getValue())) { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 21ab2139d..82c6d368d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -2352,7 +2352,7 @@ public class AddressServiceImpl implements AddressService { * @param city * @param address * @param checkBo - * @param time + * @param addressList */ private void checkAmapNew(String city, String address, AddressCheckBo checkBo, Set addressList) { try { @@ -2704,17 +2704,6 @@ public class AddressServiceImpl implements AddressService { return true; } } - } else if (null != district.getLat3() && null != district.getLng3()) { // 腾讯坐标 gps - String locations = converTenxunToAmap(formatLatLng(district.getLng3()), - formatLatLng(district.getLat3())); - if (StringUtils.isNotEmpty(locations)) { - String[] los = locations.split(","); - double distance = GeoUtils.calculateDistance(Double.valueOf(los[0]), Double.valueOf(los[1]), lng, - lat, GaussSphere.WGS84); - if (distance <= DISTANCE_RANGE) { - return true; - } - } } } catch (Exception e) { LOG.error("判断小区距离出错", e); @@ -3571,7 +3560,7 @@ public class AddressServiceImpl implements AddressService { * * @param city * @param address - * @param checkBo + * @param keywordsFormat */ private AddressCheckTruth checkTruthSplit(String city, String address, Map keywordsFormat) { // 1.拆分 @@ -4581,7 +4570,7 @@ public class AddressServiceImpl implements AddressService { /** * 地址评级具体逻辑 * - * @param checkBo + * @param checkTr * @return */ private AddressCheckTruth gradeAddressTruth(AddressCheckTruth checkTr) { @@ -4605,7 +4594,7 @@ public class AddressServiceImpl implements AddressService { /** * 小区评分 * - * @param checkBo + * @param check * @return */ private boolean gradeDistrictTruth(AddressCheckTruth check,boolean isConsume) { diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png index 366fdf4f2..654b2bce2 100644 Binary files a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/cpdb_img01.png differ diff --git a/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java b/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java index 1aacf834b..3238d95f8 100644 --- a/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java +++ b/projects/fangjia-youda/fangjia-youda-web-customer/src/main/java/com/fangjia/youda/controller/HouseToCController.java @@ -309,7 +309,7 @@ public class HouseToCController { params.put("token", token); params.put("size", query.getArea()); params.put("app", "youda");// open淇濆瓨浼颁环璁板綍 - params.put("youDa", true); + //params.put("youDa", true); if (houseInfo.getFloor() != 0 && houseInfo.getHeight() != 0) { params.put("floor", houseInfo.getFloor()); params.put("maxFloor", houseInfo.getHeight()); @@ -601,7 +601,7 @@ public class HouseToCController { params.put("name", name); params.put("type", type * (-1)); params.put("evalDate", DateUtils.date2Str(houseInfo.getEvalTime())); - params.put("youDa", true); + //params.put("youDa", true); DistrictEvaluateSingleDto result = openApiProxyService.getProxyService(FshApiRemoteService.class) .getFshDistPriceTrends(params);