diff --git a/Jenkinsfile b/Jenkinsfile index 37a7f1e92..b8b0cf0b7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent any parameters { - choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '请选择å�‘布的项目') + choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'], description: '请选择å�‘布的项目') } tools { jdk 'JDK8' @@ -17,12 +17,12 @@ pipeline { stage('Maven Build') { steps { script { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'] for (def project : projects) { if (params.PROJECT_NAME == project) { dir("deliverables/${project}") { withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { - sh 'mvn clean package -Dmaven.test.skip=true' + sh 'mvn clean package -Dmaven.test.skip=true' } } } @@ -33,8 +33,8 @@ pipeline { stage('Docker Build') { steps { script { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] - def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'] + def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"] for (def project : projects) { if (params.PROJECT_NAME == project) { dir("projects/${imagesName[project]}") { @@ -53,8 +53,8 @@ pipeline { stage('K8S Deploy') { steps { script { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] - def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'] + def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"] for (def project : projects) { if (params.PROJECT_NAME == project) { dir("deliverables/k8s") { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index f998411a0..ba0985a47 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1861,6 +1861,44 @@ public class AddressController { return mav; } + /** + * ÕÐÉÌ֤ȯPOC²âÊÔ½Ó¿Ú·â×° + */ + public Object cmbTestAction(AddesssMatchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("µØÖ·²»ÄÜΪ¿Õ")); + return mav; + } + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/cmbTest", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/cmbTest", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("·þÎñÉý¼¶ÖÐ...")); + } + return mav; + } + /** * Îå¼¶µØÖ·±ê×¼»¯½Ó¿Ú * diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index 1efa12d0f..5f2fddf88 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -208,7 +208,7 @@ public class HouseController { } - @Authority(role = Role.USER) + @Authority(role = Role.NONE) public Object similarAction(@Valid CollateralSampleQuery collateralSampleQuery){ RestfulModelAndView mav = new RestfulModelAndView(); @@ -253,7 +253,7 @@ public class HouseController { /**ÉÌÆÌÏàËÆ·¿Ô´½Ó¿Ú*/ - @Authority(role = Role.USER) + @Authority(role = Role.NONE) public Object shopSimilarAction(@Valid ShopCollateralSampleQuery query){ RestfulModelAndView mav = new RestfulModelAndView(); ShopCollateralSampleQueryBean queryBean = new ShopCollateralSampleQueryBean(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java index 2c7a22187..998b4664e 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java @@ -39,7 +39,7 @@ public class JnBankController { private static final Pattern STAGE_PATTERN = Pattern.compile(".*[\\w¶«Äϱ±ÖÐÎ÷¼×ÒÒ±û¶¡Ò»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]+(ÆÚ|Çø)$"); - @Authority(role = Role.USER) + @Authority(role = Role.NONE) public Object searchAction(@Valid JNBankSearchModel model){ RestfulModelAndView mav = new RestfulModelAndView(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index cd0d1a45b..4edbf78a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //Èںϵ¼ÊýʱʹÓà -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //¹ÀÖµµ¼ÊýʱʹÓà +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //Èںϵ¼ÊýʱʹÓà + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //¹ÀÖµµ¼ÊýʱʹÓà def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,9 +78,13 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt Éú³É->µ¼Oracle" -// delPgData() -// saveToPgSql(rows) + delPgData() + saveToPgSql(rows) // ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\bigdata_community.xlsx", allDatas, "Êý¾Ý") + +// def datas = readTxt("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_COMMUNITY_GJ.dat") +// delPgData() +// saveToPgSql2(datas) } def addDeleteDist(dists, ccbMaps, rets){ @@ -255,14 +259,74 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(20000, sql){ ps-> + sqlInstance.withBatch(1, sql){ ps-> for(def r : rows) { - ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) - ps.toString() + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } } } println "end: "+new Date() } + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + def index = 0 + sqlInstance.withBatch(10000, sql){ ps-> + for(def r : newRows) { + index++ + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + index + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 5e7d45516..e00fdeb41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{ } println "curDate:"+curDate - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } - } -// def rows = [] -// def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 // for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="ÉϺ£"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt Éú³É " + rows.size() -// -// saveToPgSql(rows) + def rows = [] + def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="ÉϺ£"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt Éú³É " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 78e552ea5..db2c71d84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230318, 20230419, houseIdsfilter); //²âÊÔͨ¹ýÆ¥Åä½Ó¿ÚÅ䷿ԴЧ¹û // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 9515d1d1b..845b06548 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -146,7 +146,9 @@ class CcbCommonUtils { public static def findGuidePrice(city, startDate, endDate){ def q = new BasicDBObject() q.put("city", city) - q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); + if(startDate && endDate){ + q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); + } def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q) def rets = [] for(def d : dbcursor){ @@ -428,7 +430,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","Ãù´äÀ¶Íå") +// q.put("name","½õÄÏÆÖÈýÐ¡Çø") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["ÆÕͨסլ","ÉÌס"])) @@ -660,4 +662,13 @@ class CcbCommonUtils { pgsqlInstance.executeUpdate(sql2) } + + public static def findDistGovPrice(def city){ + def ids = [] + pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it-> + ids << it.gujia_bds_id + }); + return ids; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 4ec047367..716114623 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2023-02-27", "2023-03-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) //¸üо²Ì¬Êý¾Ý±í -// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 94d42d3a3..84bbeaa2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202302//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202212 - def endDate = 202302 + def sDate= 202303//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202301 + def endDate = 202303 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -46,6 +46,15 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { + +// def gocPrices = CcbCommonUtils.findDistGovPrice("ÉîÛÚ") +// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?" +// pgsqlInstance.withBatch(500, sql){ ps-> +// for(def id : gocPrices){ +// ps.addBatch(id) +// } +// } + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 @@ -53,13 +62,20 @@ class ThirdPartyPriceBatch extends BaseBatch { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ + + def gocIdsPrices = [] + if(city == "ÉîÛÚ"){ + gocIdsPrices = CcbCommonUtils.findDistGovPrice(city) + } + def statMap = getDistrictStatByCity(city, weekend) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) def thirdPriceMap = findThirdPartyPrice(city, statMap) //def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//ÀϵÚÈý·½¾ù¼Û±í£¨ÏÖ²»Óã© - def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap) + + def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices) // def rets = createData3(cityCodeDatas.get(city))//еÚÈý·½¾ù¼Û±í£¬Òª×ö2²½£¬Âé·³ rows.addAll(rets) println index+" "+city+" "+rets.size() +" "+rows.size() @@ -187,10 +203,12 @@ class ThirdPartyPriceBatch extends BaseBatch { } - def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){ def allRows = [] for(def d : dists){ - + if(gocPrices && gocPrices.contains(d._id.toString())){ + continue; + } def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ def data = distCodeMap.get(d.region) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index e25da703d..ce4355ead 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -66,10 +66,7 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) - //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) - exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) + //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý // exportCommunityBasePrice(lastMonth) @@ -78,16 +75,22 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataSupplierStatistics(lastMonth) // exportBigdataModelingBase(lastMonth) - exportPgCommunityBasePrice(lastMonth) - exportPgCommunityPdescribeGj(lastMonth) - exportPgGuideStatistics(lastMonth) - exportPgBigdataSupplierStatistics(lastMonth) - exportPgBigdataModelingBase(lastMonth) - //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//Èںϰ´·¢²¼Ê±¼äµ¼£¬¹ÀÖµ°´updatetimeµ¼³ö } + if(curDate.toString().endsWith("28")){ + exportPgBigdataSupplierStatistics(lastMonth) + //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) + exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) + exportPgCommunityBasePrice(lastMonth) + exportPgCommunityPdescribeGj(lastMonth) + exportPgBigdataModelingBase(lastMonth) + exportPgGuideStatistics(lastMonth) + } + SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index d4d8007e7..45da16ec1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -50,6 +50,7 @@ class CityInfoAnaBatch extends BaseBatch { final static DBCollection HOUSE_LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia", "house_loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000); + final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo) static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616') final static DSCollection SEAWEED = DbUtils.seaweed public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) @@ -62,35 +63,41 @@ class CityInfoAnaBatch extends BaseBatch { car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', - alias:'alias',addr_alias:'addrAlias' + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount' ] public static final def OFFICE_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', - all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', - car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure', - facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics' + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure', + facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',licence:'licence', + sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',estate_charge:'estateCharge', + floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife', + conditioner_time:'conditionerTime',elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount', + goods_elevator_count:'goodsElevatorCount',network_way:'networkWay',fitment_level:'fitmentLevel',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea', + is_foreign:'foreign',is_register:'register',lighting_ventilation:'lightingVentilation',lobby_height:'lobbyHeight' ] - + public static final def SHOP_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany', rent_type :'rentType',lease_term :'leaseTerm',car_site :'carSite',estate_charge :'estateCharge',divisible :'divisible',floor :'floor', finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate', bay_area :'bayArea', property_life :'propertyLife',unified_manage :'unifiedManage',licence :'licence',business_status :'businessStatus', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',bed_kind:'bedKind',floor_info:'floorInfo',car_site_count :'carSiteCount', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics' + ] public static CITY_GRADE_MAP = [1000: "Ò»Ïß" , 1500 : "ÐÂÒ»Ïß" , 2000 : "¶þÏß" , 3000:"ÈýÏß" ,4000 : "ËÄÏß" ,5000 : "ÎåÏß"] final static def COUNT_CITYS = ["³üÖÝ","»ÆÉ½","Âí°°É½","Îߺþ","ËÞÖÝ","Ðû³Ç","±±¾©","°×Òø","¶¨Î÷","¸ÊÄÏ","½ð²ý","À¼ÖÝ","ÁÙÏÄ","¤ÄÏ","ƽÁ¹","ÇìÑô","ÌìË®","ÎäÍþ","ÕÅÒ´","¶«Ý¸","·ðɽ","¹ãÖÝ","»ÝÖÝ","½­ÃÅ","÷ÖÝ","ÇåÔ¶","ÉîÛÚ","ÓñÁÖ","°²Ë³","¹óÑô","Í­ÈÊ","ÈýÑÇ","º£¿Ú","³ÐµÂ","ºªµ¦","ºâË®","ÀÈ·»","ÇØ»Êµº","ʯ¼Òׯ","¿ª·â","ÂåÑô","´óÇì","¶õÖÝ","»Æ¸Ô","Ê®Ñß","Î人","Ò˲ý","³£µÂ","³¤É³","³£ÖÝ","»´°²","Á¬ÔƸÛ","ÄϾ©","ÄÏͨ","ËÕÖÝ","Ì©ÖÝ","ÎÞÎý","ËÞǨ","ÐìÖÝ","ÑγÇ","ÑïÖÝ","Õò½­","¾°µÂÕò","¾Å½­","ÉÏÈÄ","ӥ̶","¸§Ë³","ÁÉÑô","Å̽õ","ÉòÑô","ÌúÁë","Óª¿Ú","ºôºÍºÆÌØ","ʯ×ìɽ","Òø´¨","º£¶«","Î÷Äþ","±õÖÝ","µÂÖÝ","¼ÃÄÏ","¼ÃÄþ","Çൺ","Ϋ·»","×Ͳ©","̫ԭ","±¦¼¦","μÄÏ","Î÷°²","ÏÌÑô","ÉϺ£","³É¶¼","´ïÖÝ","µÂÑô","ÀÖɽ","Á¹É½ÖÝ","ãòÖÝ","üɽ","ÃàÑô","Äϳä","ÄÚ½­","ËìÄþ","ÑŰ²","Ò˱ö","×ÊÑô","×Ô¹±","Ìì½ò","Àö½­","ÕÑͨ","º¼ÖÝ","ºþÖÝ","¼ÎÐË","½ð»ª","ÀöË®","Äþ²¨","áéÖÝ","ÉÜÐË","̨ÖÝ","ÎÂÖÝ","ÖÛɽ","ÖØÇì"]; + final static def DIST_FACILITY_LIST = ["bank", "hospital", "bus", "shop", "university", "middle_school", "primary_school", "kindergarten", "food_shop", "supermarket","atm","school","subway"] /** * ͳ¼Æ½á¹û½ÓÊÕÓÊÏäÁбí */ final static def receivers = [ - "puxueli@fangjia.com", - "wangchencen@fangjia.com", + "gaojuan@fangjia.com", "caoyuyan@fangjia.com", "muliangshu@fangjia.com" ] @@ -116,7 +123,9 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> - CITYS << k.toString(); +// if(["ÉϺ£","ÉîÛÚ","±±¾©","¹ãÖÝ","º¼ÖÝ","ËÕÖÝ","ÖØÇì","³É¶¼"].contains(k.toString())){ + CITYS << k.toString(); +// } } // ËùÓгÇÊеÄÈË¿ÚÐÅÏ¢ @@ -146,7 +155,7 @@ class CityInfoAnaBatch extends BaseBatch { allCityDisCount.put(it.name, it.count.intValue()); } } - + // ³ÇÊÐÐÂÅÌÊý def allCityNewDisCount = [:]; disQuery = new BasicDBObject(); @@ -162,7 +171,7 @@ class CityInfoAnaBatch extends BaseBatch { allCityNewDisCount.put(it.name, it.count.intValue()); } } - + // ³ÇÊÐд×ÖÂ¥Êý def allCityOfficeCount = [:]; disQuery = new BasicDBObject(); @@ -178,8 +187,8 @@ class CityInfoAnaBatch extends BaseBatch { allCityOfficeCount.put(it.name, it.count.intValue()); } } - - + + // ³ÇÊÐÉÌÆÌÊý def allCityShopCount = [:]; disQuery = new BasicDBObject(); @@ -503,7 +512,7 @@ class CityInfoAnaBatch extends BaseBatch { entity.put('official_population', curInfo.official_population); entity.put('district_avg_population', curInfo.district_avg_population); entity.put('rate', curInfo.rate); - + removeCurInfo(cityName, weekend); } else { def cityHouseStat = allHouseStats.get(cityName); @@ -513,13 +522,13 @@ class CityInfoAnaBatch extends BaseBatch { } else if (cityHouseStat && cityHouseStat.standard_people_count) { entity.put('official_population', cityHouseStat.standard_people_count); } - + if (lastInfo && lastInfo.district_avg_population) { entity.put('district_avg_population', lastInfo.district_avg_population); } else if (cityHouseStat && cityHouseStat.district_avg_population) { entity.put('district_avg_population', cityHouseStat.district_avg_population); } - + if (lastInfo && lastInfo.rate) { entity.put('rate', lastInfo.rate.toDouble()); } else if (cityHouseStat) { @@ -565,6 +574,8 @@ class CityInfoAnaBatch extends BaseBatch { entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4)) } entity.put('district_detail', districtDetail(disCount, cityName)); + + entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName)); } else { entity.put('district_detail', [:]); } @@ -1079,6 +1090,75 @@ class CityInfoAnaBatch extends BaseBatch { districtDetail.put('rentPrice', roundDouble(pc, disCount)); return districtDetail; } + + def districtFacilitysDetail(def disCount, def cityName){ + DBObject query = new BasicDBObject(); + query.put('city', cityName); + query.put('cat', 'district'); + query.put('visible', 0); + + DBObject fields = new BasicDBObject() + fields.put("city", 1) + fields.put("region", 1) + fields.put("name", 1) + + def facilityMap = findFacilityByCity(cityName) + DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) + Iterator iter = cur.iterator() + def countMap = [:] + + while(iter.hasNext()) { + def dbo = iter.next() + def facilityCountMap = facilityMap.get(dbo.city+"##"+dbo.region+"##"+dbo.name) + if(facilityCountMap){ + DIST_FACILITY_LIST.each() { + def count = facilityCountMap.get(it) + if (count) { + if(countMap.containsKey(it)) { + def tempCount = countMap.get(it) + 1 + countMap.put(it, tempCount) + }else { + countMap.put(it, 1) + } + } + } + } + } + + def facilityDetail = [:]; + DIST_FACILITY_LIST.each() {it-> + if(countMap.containsKey(it)) { + facilityDetail.put(it, roundDouble(countMap.get(it), disCount)) + }else { + facilityDetail.put(it, 0) + } + } + return facilityDetail; + } + + def findFacilityByCity(city){ + DBObject query = new BasicDBObject(); + query.put('city', city); + DBCursor cur = FACILITY_CONN.find(query).batchSize(2000) + Iterator iter = cur.iterator() + def distMap = [:] + while(iter.hasNext()) { + def dbo = iter.next() + def countMap = [:] + DIST_FACILITY_LIST.each { + def values = dbo.get(it) + def count = 0 + if(values){ + count = values.size() + countMap.put(it, count) + } + } + if(countMap){ + distMap.put(dbo.city+"##"+dbo.region+"##"+dbo.district_name, countMap) + } + } + return distMap; + } double roundDouble(int a, int base) { if(a == 0 || base == 0) { @@ -1177,7 +1257,7 @@ class CityInfoAnaBatch extends BaseBatch { DBObject query = new BasicDBObject(); query.put('city', cityName); query.put('cat', 'office'); - query.put('visible', 0); + query.put('status', 0); DBObject fields = new BasicDBObject() OFFICE_FIELDS_MAP.each() {key,skey-> @@ -1187,7 +1267,7 @@ class CityInfoAnaBatch extends BaseBatch { DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) Iterator iter = cur.iterator() def countMap = [:] - + println cur.count() while(iter.hasNext()) { def dbo = iter.next() OFFICE_FIELDS_MAP.each() {key,skey-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index ac8449aa8..f7dcc4293 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -124,7 +124,7 @@ class ThirdPriceExportBatch extends BaseBatch{ add = true; } - def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice) + def tc_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)//58tc_esf_price row << tc_price if(tc_price){ add = true; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy new file mode 100644 index 000000000..231a930c8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -0,0 +1,530 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import groovy.sql.Sql + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.Statement +import java.util.regex.Pattern + +class OfficeProcess { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + def sqlInstance = Sql.newInstance("jdbc:clickhouse://192.168.0.216:8123/etl?max_memory_usage=50000000", + "goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver") + + static main(args) { + def filePath = "/home/goojia/gj/6³ÇδƥÅäÉϵÄд×ÖÂ¥.xlsx" + + if(CommonDatautils.isWindow()){ + filePath = "D:\\Êý¾Ý\\д×ÖÂ¥\\6³ÇδƥÅäÉϵÄд×ÖÂ¥.xlsx" + } + def task = new OfficeProcess(); + +// task.run("D:\\Êý¾Ý\\д×ÖÂ¥\\6³ÇºÏ²¢Ð´×ÖÂ¥.xlsx") +// task.addPropertyType2("ÉîÛÚ") + +// task.standardName(filePath) +// task.nameRepeatFiter(filePath) +// task.addressFiter(filePath) +// task.nameFiter(filePath) +// task.typeFiter(filePath) +// task.nameFlag(filePath) +// task.updateFlagToDB(filePath) +// task.updateFlagToDB2(filePath) + + task.checkRegion() + + } + + def nameRepeatFiter(filePath){ + def keys = [] + ExcelUtils.modify(filePath, { row -> + if (!row['L'] && row['B']) { + def city = row['B'] + def region = row['C'] + def name = "" + if(row['K']){ + name = row['K'] + }else{ + name = row['D'] + } + def key = city+"##"+region+"##"+name + if(keys.contains(key)){ + row['L'] = "Ãû³ÆÈ¥ÖØ" + println key + }else{ + keys << key + } + } + }) + } + + def nameFiter(filePath){ + ExcelUtils.modify(filePath, { row -> + def name = row['K'] + + if(!row['L'] && !name){ + row['L'] = "Ãû³Æ¹ýÂË" + } + if(!row['L'] && name){ + if(name.contains("¹«Ë¾")){ + row['L'] = "Ãû³Æ¹ýÂË" + } + } + + if(!row['L'] && name){ + if(isStandardRoad(name)){ + row['L'] = "Ãû³Æ¹ýÂË" + } + } + + + + if(!row['L'] && name){ + def fiters = ["³§·¿","²âÊÔ", "·þÎñ","×ۺϷþÎñ","¾Æµê","ÉîÛÚÌØ×¨","¹¤Òµ","¾­¼Ã","²úÒµ","¾Óί»á","ËÞÉá","¹ºÎï","ÐÝÏÐ","ÎÄ»¯","ѵÁ·","³ö×â","Õã²Ë","Öµ°à¸Ú","ÕÐ×â","ÉÌס","°ì¹«ÊÒ","ÅäÌ×","ÃÀʳ","²©Îï¹Ý","ÎïÁ÷","ÉÌÆÌ","¹«Éç","É̳Ç","Öµ°àÊÒ","ÔÝÍ£","´´ÒµÉçÇø","ÕÐÉÌ","±ö¹Ý","Æ÷²Ä","½ÌÓý","ÃÅÕï","¸¾²ú¿Æ","ÉÌó³Ç","µç×Ó³Ç","³¬ÊÐ","ÁãÊÛ"] + for(def key : fiters){ + if(name.contains(key)){ + row['L'] = "Ãû³Æ¹ýÂË" + break; + } + } + } + + if(!row['L'] && name){ + def fiters = ["´óÏÃ","ȹ¥",'»¨Ô°','´óÂ¥','´óÌÃ','ÄÏËþ','±±¾©','´óµÀ','Ö÷Â¥','Öйú','ÖÐÐÄ','¹ã³¡','Äñ³²','Ð¡Çø','µ¥Î»','°ì¹«','±±Ëþ' + ,'д×ÖÂ¥','¼ìÑéÂ¥','°ì¹«Â¥','½Ó´ýÇø','ÉÌÎñÂ¥','ÎÄÌåÂ¥','ÊÐÖÐÐÄ','²úÒµÔ°','¿Æ¼¼Â¥','ÐÐÕþÂ¥','²¡·¿43¡¢44¡¢45','ÂɼºÂ¥','µ÷¶ÈÂ¥','»ÔÌÚÂ¥','×ÜÎñÂ¥' + ,'7´±»õÌÝ','ÏÖ³§B¶°','(ÂíÁ¬µÀÄϽÖ)','ÖÐÑëµçÊǪ́','ºó×ù','Î÷´óÂ¥','B&C','ÉÌÒµÓÃÂ¥','²ÆÎñÖÐÐÄ','µØÌú','һƿ','Íâó¥','ÒµÎñÂ¥','Ñз¢´óÂ¥','½áËãÖÐÐÄ','Âí³¡','ÉÌÎñ'] + for(def key : fiters){ + if(name == key){ + row['L'] = "Ãû³Æ¹ýÂË" + break; + } + } + } + + + if(!row['L'] && row['J'] && name){ + if(row['J'].toString().contains("ÉÌÎñд×ÖÂ¥") && name.contains("¹«Ô¢")){ + row['L'] = "Ãû³Æ¹ýÂË" + } + } + + if(!row['L'] && name){ + if(findPattern(name,'^[A-Za-z0-9]+$') && name.toString().length()<=3){ + row['L'] = "Ãû³Æ¹ýÂË" + }else if(name.toString().length()<2){ + row['L'] = "Ãû³Æ¹ýÂË" + } + } + + if(!row['L'] && name){ + def filters = ['³ï±¸×é','Ð¡Çø','¼ÒÊô·¿','¼ÒÊôÔº','¼ÒÊôÂ¥','×Ô±à','Êг¡','ÏîÄ¿','Åú·¢³Ç','´«´ïÊÒ','ÌÝ','ÐͲÄ','ѧԺ','¹¤µØ','Àä¶³³§','»ú¹ØÂ¥','ÔĶÁÊÒ','ÒéÊÂÌü','µçÄÔ³Ç','ͯװ³Ç','ÅɳöËù','±£°²ÊÒ','É¢ÅÌ','µê','¹¤×÷Õ¾',"Îïҵ¥",'ÎïÒµ','æäÕ¾','¹øÂ¯·¿','´¦ÀíÖÐÐÄ',"ÏúÊÛÖÐÐÄ",'Õþ¸®','¹Ý','¾èÔùµã','°Ù»õ','¹¤³§','¹¤³¡','¼ì²â³¡','Ö°¹¤Ö®¼Ò','ÒÆ¶¯','ÖÐÐÄÕ¾','Â¥ÃÅ','Ö°³¡','ÏÊ»¨»ùµØ','Ò½Ôº','×°ÊγÇ','ʵѵ¥','ÒµÎñÂ¥','¿ÆÑÐÂ¥','ѧУ','½»Ò׳Ç','¹ÜÀíÂ¥','¿¼³¡','ͨµÀ','ÍŹº','Â¥ÉÏ','ÔËÓªÖÐÐÄ','ÉÌÒµ³Ç'] + for(def key : filters){ + if(name.toString().endsWith(key)){ + row['L'] = "Ãû³Æ¹ýÂË" + break; + } + } + } + }) + } + + def typeFiter(filePath){ + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['J']){ + def fiters = ["ÉÌס","·þÎñ","×¡Õ¬Ð¡Çø","¾Æµê","ÉÌÒµ×ÛºÏÌå","סլµ×ÉÌ"] + for(def key : fiters){ + if(row['J'].toString().contains(key)){ + row['L'] = "ÀàÐ͹ýÂË" + break; + } + } + } + if(!row['L'] && row['B'] && !row['J']){ + row['L'] = "ÀàÐͿհ׹ýÂË" + } + }) + } + + def nameFlag(filePath){ + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['D']) { + def fiters = ["´óÏÃ","´óÂ¥","ÉÌÎñÂ¥","ÉÌÎñÖÐÐÄ","¹ã³¡","д×ÖÂ¥","°ì¹«Â¥","¿Æ¼¼Ô°","¹ú¼Ê´óÏÃ","ÉÌÎñ´óÏÃ","ÉÌÎñ¹ã³¡","´´Ô°","´´Òâ","ÖÐÐÄ","ÓªÏúÖÐÐÄ"] + for(def key : fiters){ + if(row['D'].toString().contains(key)){ + if(row['D']=="¹ã³¡"){ + + } + row['L'] = "Ãû³Æ¹Ø¼ü×Ö[Ìí¼Ó]" + break; + } + } + } + + if(!row['L'] && row['J']){ + if(row['J'].toString().contains("Ô°Çø") || row['J'].toString().trim()=="´¿Ð´×ÖÂ¥" || row['J'].toString().trim()=="±ê׼д×ÖÂ¥"){ + row['L'] = "ÀàÐÍ[Ìí¼Ó]" + } + + if(!row['L'] && row['J'].toString().trim()=="´¿Ð´×ÖÂ¥ ±ê׼д×ÖÂ¥" || row['J'].toString().trim()=="ÉÌÎñÖÐÐÄ" || row['J'].toString().trim()=="ÉÌÎñÖÐÐÄ ±ê׼д×ÖÂ¥"){ + row['L'] = "ÀàÐÍ[Ìí¼Ó]" + } + } + + if(!row['L'] && row['J'] && row['D']){ + if(row['J'].toString().contains("ÉÌÎñд×ÖÂ¥") && !row['D'].toString().contains("¹«Ô¢")){ + row['L'] = "ÀàÐÍ[Ìí¼Ó]" + } + } + + if(!row['L'] && row['J'] && row['D']){ + if(row['J'].toString().contains("´¿Ð´×ÖÂ¥")){ + row['L'] = "ÀàÐÍ[Ìí¼Ó]" + } + } + + }) + + } + + def addressFiter(filePath){ + def keyMap = [:] + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['F']){ + def city = row['B'] + def region = row['C'] + def name = "" + if(row['K']){ + name = row['K'] + }else{ + name = row['D'] + } + def address = row['F'] + def key = city+"##"+region+"##"+address + def keys = keyMap.keySet() + if(keys.contains(key)){ + def values = keyMap.get(key) + if(values){ + for(def n : values){ + def score = CosineSimilarAlgorithm.getSimilarityRatio(name, n) + if(score>=0.8){ + row['L'] = "µØÖ·È¥ÖØ" + break; + } + println address +" "+ name+" "+n +" "+score + } + } + + }else{ + def values = keyMap.get(key) + if(!values){ + values = [] + keyMap.put(key, values) + } + values << name + } + } + + }); + } + + def standardName(filePath){ + def index = 0 + ExcelUtils.modify(filePath, { row -> + index++ + if(!row['K'] && row['D']){ + def name = row['D'] + def newName = name.replaceAll("-|¡¤|(|)|//|\\?|¡°|¡±","").replaceAll("\\w+´±\\w+×ù","").replaceAll(" ","").replaceAll("\\.","").replaceAll("Ø­","") + newName = newName.replaceAll("-?\\w+ºÅÉÌҵ¥","").replaceAll("\\?","") + newName = newName.replaceAll("\\w+Â¥\\w+ÊÒ","") + newName = newName.replaceAll("\\w+ºÅ\\w+(×ù|´±)","") + newName = newName.replaceAll("(Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®)ºÅÂ¥(Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®)µ¥Ôª","") + newName = newName.replaceAll("(Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|ʮһ)?´±?(Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|¶«|ÄÏ|Î÷|±±)(ËþÂ¥|Â¥)","") + newName = newName.replaceAll("-?(µÚ)?\\w+(ºÅд×ÖÂ¥|ºÅÂ¥|ºÅ´óÂ¥|Â¥|×ù|¶°|ÄÏÂ¥|±±Â¥|¶«Â¥|Î÷Â¥|Çø|µ¥Ôª)","") + newName = newName.replaceAll("(Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|Ò¼|·¡|Èþ|ËÁ|Îé|½|Æâ|°Æ|¾Á|ʰ)¶°","") + newName = newName.replaceAll('(¶«|ÄÏ|Î÷|±±)×ù$',"") + newName = newName.replaceAll("1ÆÚ","Ò»ÆÚ").replaceAll("2ÆÚ","¶þÆÚ").replaceAll("3ÆÚ","ÈýÆÚ").replaceAll("4ÆÚ","ËÄÆÚ").replaceAll("5ÆÚ","ÎåÆÚ").replaceAll("6ÆÚ","ÁùÆÚ").replaceAll("7ÆÚ","ÆßÆÚ").replaceAll("8ÆÚ","°ËÆÚ").replaceAll("9ÆÚ","¾ÅÆÚ").replaceAll("10ÆÚ","Ê®ÆÚ").replaceAll("1-5ÆÚ","Ò»-ÎåÆÚ").replaceAll("5-6ÆÚ","Áù-ÆßÆÚ") + newName = newName.toUpperCase().replaceAll("(×°ÐÞÖÐ)","").replaceAll("\\(\\)","") + + if(newName){ + def md = DataUtil.match(newName, Pattern.compile("^[0-9|a-z|A-Z|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|Áã|Ò¼|·¡|Èþ|ËÁ|Îé|½|Æâ|°Æ|¾Á|ʰ|¶«|ÄÏ|Î÷|±±|ÖÐ]{0,1}[д×Ö|°ì¹«|Â¥|×ù|Çø|¶°|×é|´±|0-9|¶«|ÄÏ|Î÷|±±|ÖÐ|ºÅ]?[0-9|a-z|A-Z|Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|Áã|Ò¼|·¡|Èþ|ËÁ|Îé|½|Æâ|°Æ|¾Á|ʰ|¶«|ÄÏ|Î÷|±±|ÖÐ|°ì¹«|д×Ö]?[Â¥|×ù|Çø|¶°|×é|´±|0-9|¶«|ÄÏ|Î÷|±±|ÖÐ]\$")) + if(md){ + newName = '' + } + } + row['K'] = newName + } + }) + } + + def addPropertyType2(runCity){ + def map = [:] + def datas = ExcelUtils.read("D:\\Êý¾Ý\\д×ÖÂ¥\\6³ÇδƥÅäд×ÖÂ¥.xlsx",0,1) + if(datas){ + for(def d : datas){ + def key = d['B']+"##"+d['C']+"##"+d['D'] + def list = map.get(key) + if(!list){ + list = [] + map.put(key, list) + } + if(d['M'] && !list.contains(d['M'])){ + list << d['M'] + } + if(d['U'] && !list.contains(d['U'])){ + list << d['U'] + } + + } + } + + def index = 0 + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\д×ÖÂ¥\\6³ÇδƥÅäÉϵÄд×ÖÂ¥.xlsx", { row -> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def types = map.get(city+"##"+region+"##"+name) + if(types){ + row['J'] = types.join(" ") + } + }) + } + + def addPropertyType(){ + def index = 0 + ExcelUtils.modify("D:\\Êý¾Ý\\д×ÖÂ¥\\δƥÅäд×ÖÂ¥.xlsx", { row -> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def types = findDatasByDB(city, region, name) + if(types){ + row['J'] = types.join(" ") + } + println index + }) + } + + def findDatasByDB(city, region, name){ + def sql = "select * from office where city='${city}' and region='${region}' and name='${name}'" + def types = [] + sqlInstance.eachRow(sql, {r-> + if(r.property_type && !types.contains(r.property_type)){ + types << r.property_type + } + }) + return types + } + + def run(filepath){ + def index = 0 + ExcelUtils.multiThreadingModify(filepath,{row-> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def address = row['I'] + if(city ){ + def jsonObject = OpenApiClient.addressMatch(city, name, "office", false, region, null) +// if(jsonObject && jsonObject.precise == -1){ +// jsonObject = OpenApiClient.addressMatch(city, address, "office", false, region, name) +// } + if (jsonObject != null && jsonObject.precise != -1) { + row['J'] = jsonObject.get("id") + row['K'] = jsonObject.get("city") + row['L'] = jsonObject.get("district") + row['M'] = jsonObject.get("name") + row['N'] = jsonObject.get("credit") + row['O'] = jsonObject.get("propertyType") + } + } + println index + }) + } + + def isStandardRoad(def s){ + def pattern = Pattern.compile("[·|½Ö|´óµÀ|Ïï]\\d+(ºÅ|Ū)?"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + if(s.toString().endsWith("·")||s.toString().endsWith("½Ö")){ + return true; + } + return false + } + + + def findPattern(def address, def regex) { + Pattern pattern = Pattern.compile(regex); + def matcher = pattern.matcher(address) + if (matcher.find()) { + return true + } + return false; + } + + def updateFlagToDB(filePath){ + def datas = ExcelUtils.read(filePath,0,1) + def total = datas.size() + println total + def index = 0 + def rets = [] + for(def d : datas){ + def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';" + index++ + rets << sql + //sqlInstance.executeUpdate(sql) + //println index+"/"+ total + println sql + } + + + def filePath1 = "d:\\Data1.txt" + File file = new File(filePath1) + def s = new StringBuffer() + for(def r : rets){ + s.append(r+"\n") + } + file.write(s.toString()) + } + + + def updateOption(city, region, name, source, flag){ + def sql = "alter table office update region_fence='${flag}' where city='${city}' and region='${region}' and name='${name}' and source='${source}';" + sqlInstance.executeUpdate(sql) + } + + def batchUpdate(rets){ + // ³õʼ»¯Çý¶¯ + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + // url + String url = "jdbc:clickhouse://192.168.0.216:8123/etl"; + // Óû§ÃûÃÜÂë + String user = "goojia"; + String password = "goojia123456"; + // µÇ¼ + Connection connection = DriverManager.getConnection(url, user, password); + Statement stmt = connection.createStatement(); + def count = 0 + def sql = "alter table office update region_fence=? where city=? and region=? and name=? and source=?;" + PreparedStatement pstmt = connection.prepareStatement(sql) + + for(def d : rets){ + count++ + pstmt.setString(1, d.flag) + pstmt.setString(2, d.city) + pstmt.setString(3, d.region) + pstmt.setString(4, d.name) + pstmt.setString(5, d.source) + println count+" "+d.city+" "+d.name + pstmt.addBatch() + if(count % 1 ==0) { + try { + pstmt.executeBatch() + } catch (Exception e) { + e.printStackTrace() + } + } + } + pstmt.executeBatch() + } + def updateFlagToDB2(filePath){ + // ³õʼ»¯Çý¶¯ + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + // url + String url = "jdbc:clickhouse://192.168.0.216:8123/etl"; + // Óû§ÃûÃÜÂë + String user = "goojia"; + String password = "goojia123456"; + // µÇ¼ + Connection connection = DriverManager.getConnection(url, user, password); + Statement stmt = connection.createStatement(); + def count = 0 + def datas = ExcelUtils.read(filePath,0,1) + def sql = "alter table office update option=?, format_name=? where city=? and region=? and name=?;" + PreparedStatement pstmt = connection.prepareStatement(sql) + + for(def d : datas){ + if(d['B']){ + count++ + pstmt.setString(1, d['L']) + pstmt.setString(2, d['K']) + pstmt.setString(3, d['B']) + pstmt.setString(4, d['C']) + pstmt.setString(5, d['D']) + println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D'] + pstmt.addBatch() + if(count % 100 ==0) { + pstmt.executeBatch() + } + } + } + pstmt.executeBatch() + } + + def checkRegion(){ + def citys = ["±±¾©","¹ãÖÝ","ÖØÇì","Ìì½ò","³É¶¼","ÄϾ©"]//"ÉϺ£","ÉîÛÚ", + for(def city : citys){ + def regionPointsMap = findRegionRange(city) + def addDatas = findAddDatasByClickhouse(city) + println "Ðè´¦ÀíµÄÇøÓòΧÀ¸£º"+addDatas.size() + def index = 0 + def rets = [] + for(def data : addDatas){ + index ++ + if(data.lng){ + def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region)) + def mark = flag?'T':'F' +// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark] + updateOption(data.city, data.region, data.name, data.source, mark) + println index+" "+data.region + " "+data.name+" "+ flag + } + } +// batchUpdate(rets) + } + } + + def findAddDatasByClickhouse(city){ + def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%Ìí¼Ó%'" + def datas = [] + sqlInstance.eachRow(sql, {r-> + datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source] + }) + return datas + } + + def findRegionRange(city){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion(city,d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index ae3119029..a254cbdd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{ // "¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï", // "ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿" // def esSources = ["ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Íò©Ê¿PC¶Ë":"Íò©Ê¿","¼ªÎÝ":"¼ªÎÝ","±´¿ÇÕÒ·¿":"±´¿ÇÕÒ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","365ÌÔ·¿":"365ÌÔ·¿","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú"] - def esSources = ["ËÑ·¿":"ËÑ·¿"] + def esSources = ["ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Íò©Ê¿PC¶Ë":"Íò©Ê¿","¼ªÎÝ":"¼ªÎÝ","±´¿ÇÕÒ·¿":"±´¿ÇÕÒ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","365ÌÔ·¿":"365ÌÔ·¿","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú"] static main(args) { @@ -43,7 +43,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = ["Á¹É½ÖÝ"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ def rets = findAllEsData(city, esSource, mgSource, mdate) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java index 604d35021..52cea9435 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java @@ -1,9 +1,6 @@ package com.fangjia.service.datastore; -import com.fangjia.core.util.BeanUtils; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.DateUtil; -import com.fangjia.core.util.HouseValidator; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.SortParam; import com.fangjia.dal.SrQuery; @@ -51,7 +48,7 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ private static String[] HOUSE_FIELDS = new String[]{"city", "region", "districtName", "totalPrice", "avgPrice", "area", "fitment", "direction", "floor", "height", "publishDate", "address", "estateType"}; - + public static String[] TAGS =new String[]{"¶À¶°", "ÁªÅÅ", "µþ¼Ó", "˫ƴ", "»¨Ô°Ãæ»ý", "Ó¾³Ø", "±ðÊû", "´ø»¨Ô°", "Ñó·¿", "µþÆ´", "¶ÀÔº", "ÁªÌå", "΢Êû", "´óƽÊû"}; private DealPriceDao dealPriceDao; private HouseDao houseDao; @@ -191,7 +188,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ } List querySamples= new ArrayList(); List samples = findSampleByDistrict(district.getCity(), district.getRegion(), district.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); - scoreForSample(samples, target); +// scoreForSample(samples, target); + sortForSample(samples, target); for(SampleWrapper sample : samples){ sample.setProperty(queryBean.getProperty()); } @@ -205,7 +203,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ for(SampleWrapper sample : nearlySample){ sample.setProperty(d); } - scoreForSample(nearlySample, target); +// scoreForSample(nearlySample, target); + sortForSample(samples, target); querySamples.addAll(nearlySample); if(querySamples.size() > max){ break; @@ -481,6 +480,56 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ return samples; } + + private List sortForSample(List samples, final SampleWrapper targetSample){ + + if(samples.isEmpty()){ + return samples; + } +// if(samples!=null && samples.size() < max){ +// return samples; +// } + Collections.sort(samples, new Comparator() { + public int compare(SampleWrapper o1, SampleWrapper o2) { + Double d1 = Math.abs(o1.getStrctureArea() - targetSample.getStrctureArea()); + Double d2 = Math.abs(o2.getStrctureArea() - targetSample.getStrctureArea()); +// return (int) (d1 - d2); + return d1.compareTo(d2); + } + }); + + List prices = new ArrayList(); + for(SampleWrapper sample : samples){ + if(!prices.contains(sample.getUnitPrice())){ + prices.add(sample.getUnitPrice()); + if(prices.size()>=10){ + break; + } + } + } + final Integer midPrice = (int)median(prices); + Collections.sort(samples, new Comparator() { + public int compare(SampleWrapper o1, SampleWrapper o2) { + Integer d1 = Math.abs(o1.getUnitPrice() - midPrice); + Integer d2 = Math.abs(o2.getUnitPrice() - midPrice); + return d1.compareTo(d2); + } + }); + return samples; + } + + private static double median(List total) { + double j = 0; + Collections.sort(total); + int size = total.size(); + if(size % 2 == 1){ + j = total.get((size-1)/2); + }else { + j = (total.get(size/2-1) + total.get(size/2) + 0.0)/2; + } + return j; + } + /** * ²éÕÒÐ¡ÇøÑù±¾ * @param city @@ -514,8 +563,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ houseQuery.add(new QueryParam("region", region )); if(districtStat != null && districtStat.getAvgPrice() > 0){ int avgPrice = districtStat.getAvgPrice(); - houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.85 ))); - houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.15 ))); + houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.5 )));//0.85 + houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.5 )));//1.15 } BooleanQuery nameQuery = new BooleanQuery(); int c = 0; @@ -548,6 +597,21 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ int maxFloor = 0; boolean allSampleHasHeight = true; for(House house : houses){ + if(StringUtils.isNotBlank(estateType) && estateType.equals("01")){//01:ÆÕͨסլ 02:±ðÊû + if(StringUtils.isNotBlank(house.getEstateType()) && house.getEstateType().contains("±ðÊû")){ + continue; + } + if(isVilla(house.getArea(), house.getName(), house.getHeight(), house.getRoom())){ + continue; + } + }else if(StringUtils.isNotBlank(estateType) && estateType.equals("02")){ + if(house.getHeight()!=null && house.getHeight()>=10){ + continue; + } + if(house.getFloor()!=null && house.getFloor()>=10){ + continue; + } + } SampleWrapper sample = convertHouse2Sample(house); if(isAvailableSample(sample)){ @@ -859,6 +923,39 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ return null; } + /** + * ÊÇ·ñÊDZðÊû + * @return + */ + public boolean isVilla(Double area, String name, Integer height, Integer room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + public void setDealPriceDao(DealPriceDao dealPriceDao) { this.dealPriceDao = dealPriceDao; }