From fbc2698c2bc6b3a6ed5940ed865e64203f42aa0a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 20 Apr 2024 15:05:16 +0800 Subject: [PATCH 1/9] update --- .../ccbevaluation/BigdataDealBatch.groovy | 8 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../script/gj/data_handle/DevProcess.groovy | 127 ++++++++++++++---- .../FjwGuiDingPriceBatch.groovy | 2 +- 4 files changed, 113 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 9b22ad4b9..210e810a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -34,7 +34,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -60, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); @@ -68,6 +68,8 @@ class BigdataDealBatch extends BaseBatch{ for(def city : allCitys){ addDealPriceMatch(city); } +// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 + // def rows = [] // def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] @@ -679,6 +681,10 @@ class BigdataDealBatch extends BaseBatch{ return map } + def updateCrawlTradeDate(){ + + } + def addDealPriceMatch(city){ // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 83c9ec794..84f840c02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -678,7 +678,7 @@ class CcbCommonUtils { } public static def deleteGuidedPrice(def dealdate){ - def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'" + def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'" pgsqlInstance.executeUpdate(sql2) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 374ad79bd..aa7425f23 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,7 +29,8 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() + task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -37,15 +38,17 @@ class DevProcess { // task.updateCityAndDist() // task.addDevAddres1() //按添加地址中包含开发区名称的地址 // task.addDevAddres2()//添加相差2公里范围内的地址 +// task.matchDevAndAddAddress() // task.stat() // task.addNewDev() // task.createDevMapping() // task.deleteAddr() // task.synchDevTo43() - } + + def parseCityAndDist(){ ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d-> if(!d['J']){ @@ -160,11 +163,11 @@ class DevProcess { } def devSplitRoad(){ - def i = 1; - while(i<13){ + def i = 10; + while(i<11){ def fileName = "/home/goojia/gj/company-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\第二批公司-${i}.xlsx" + fileName = "D:\\数据\\数据建设\\company-${i}.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ @@ -192,30 +195,33 @@ class DevProcess { }*/ if(d['C'] && d['B']!='河北省'){ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['J'] = jsonObject.get("province") - } + if(jsonObject){ + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['K'] = jsonObject.get("city") - } + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['L'] = jsonObject.get("district") - } + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } - if(jsonObject && street && streetNumber && street!="null"){ - d['M'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['N'] = jsonObject.get("streetNumber") + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + + if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ + d['O'] = jsonObject.get("dev") } } - if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ - d['O'] = jsonObject.get("dev") - } } },0) i++ @@ -223,6 +229,47 @@ class DevProcess { } + + + def devNameSplitRoad(){ + def fileName = "/home/goojia/gj/company.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx" + } + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['P'] && d['P']=="路正常" && !d['O']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E']) + if(jsonObject){ + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + + if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ + d['O'] = jsonObject.get("dev") + } + } + + } + },0) + } + def addFlag(){ ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> if(d['H'] && d['D']){ @@ -509,4 +556,38 @@ class DevProcess { } DEVELOPMENT_ZONE_43_COLL.insert(rets) } + + + def matchDevAndAddAddress(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx",{d-> + if(d['P'] && d['P']=="路正常" && d['O']){ + def q = new BasicDBObject() + q.put("name", d['O']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + + d['Q'] = obj._id.toString() + d['R'] = obj.name + def address = d['M']+d['N'] + q.put("address", address) + def sameObj = DEV_MAPPING_CONN.findOne(q) + if(!sameObj){ + def udata = new BasicDBObject() + udata.put("province", obj.province) + if(obj.city){ + udata.put("city", obj.city) + } + if(obj.district){ + udata.put("district", obj.district) + } + + udata.put("name",obj.name) + udata.put("type", "地址") + udata.put("address", address) + DEV_MAPPING_CONN.save(udata) + } + } + } + },0); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy index 8e494f407..a528b4f9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy @@ -53,7 +53,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-01", "2022-04-01") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) From 2ea9451feea9d1374a44ef3066900356ef6141d5 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Thu, 16 May 2024 17:37:59 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E6=B7=BB=E5=8A=A0userName=E7=9A=84?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=88=B7=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/Controller.java | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java index 1c508b26b..95877519f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java @@ -55,38 +55,44 @@ public class Controller { return mav; } - /** - * 缓存刷新接口 - * @return - */ - @Authority(role = Role.NONE) - public Object refreshCacheAction(RefreshCacheModel model){ - RestfulModelAndView mav = new RestfulModelAndView(); - mav.setViewType("rest"); - Token token = FangjiaApiUtil.parse2Token(model.getToken()); - if(token == null){ - mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); - return mav; - } - final ThreadLocalData threadLocalData = ThreadLocalData.getInstance(); - threadLocalData.put(ThreadLocalKey.DisableCache, "1"); - // 在验证appKey - ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId()); - if(apiUser == null){ - mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); - return mav; - } - try { - apiUserAuthorityService.findByUserId(apiUser.getId()); - if(StringUtils.isNotBlank(model.getViewConfigUrl())){ - apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl()); - } - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功")); - return mav; - } + /** + * 缓存刷新接口 + * + * @return + */ + @Authority(role = Role.NONE) + public Object refreshCacheAction(RefreshCacheModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + Token token = FangjiaApiUtil.parse2Token(model.getToken()); + if (token == null) { + mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); + return mav; + } + final ThreadLocalData threadLocalData = ThreadLocalData.getInstance(); + threadLocalData.put(ThreadLocalKey.DisableCache, "1"); + // 在验证appKey + ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId()); + if (apiUser == null) { + mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); + return mav; + } + try { + // 获取用户 + User user = userService.findById(apiUser.getId()); + if (null != user && StringUtils.isNotBlank(user.getUserName())) { + userService.findByUserName(user.getUserName()); + } + apiUserAuthorityService.findByUserId(apiUser.getId()); + if (StringUtils.isNotBlank(model.getViewConfigUrl())) { + apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl()); + } + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功")); + return mav; + } From 72899e2bce7656ca7fd9224aba4680933ca67378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 17 May 2024 11:04:37 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E5=88=86=E6=9C=9F=E5=88=86=E5=8C=BA?= =?UTF-8?q?=E4=B8=8E=E6=9C=B1=E5=B0=8F=E5=8C=BA=E6=98=A0=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ana/FindStageDistrictBatch.groovy | 2 +- .../src/main/groovy/com/fangjia/script/util/DbUtils.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy index c2f4eaee4..8c6633b4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy @@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject import org.apache.commons.lang.StringUtils import java.util.regex.Pattern - +// 分期分区与主小区映射关系表 @BatchOwner(Watcher.WangWenYan) class FindStageDistrictBatch extends BaseBatch { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 9c6952a70..aeda5fc28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.* public class DbUtils{ public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com"; - public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log'); + //public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log'); public static final String COMMON_DB = "fangjia"; public static final int COMMON_PORT = 27017; - public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater") + //public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater") public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird") From 4b7a5aba87bc8190d51353af7da1c7250a92f181 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Fri, 17 May 2024 15:42:19 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E5=B9=BF=E5=A4=A7=E6=B6=89=E5=86=9C?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/GdBankController.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java index 9b0a563e5..4edaec18d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -112,10 +112,9 @@ public class GdBankController { // 涉农结果不一样 if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) { //有市辖建制镇, 或者城市不一样 用我们 - if (result.containsKey("city")) { - if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) { - basicModel.setResult(result); - } + if (result.containsKey("city") && ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) + && StringUtils.isNotBlank(result.getString("city"))))) { + basicModel.setResult(result); } else { if (("是").equals(result.get("countyLevelCity"))) { map.put("prefectureLevelCity", "否"); From c0786b12e2a590b0dd6b6d5358cbd66f2134a212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 17 May 2024 19:51:17 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0=E6=95=88?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddrSvcController.java | 7 +++++-- .../com/fangjia/open/controllers/GdBankController.java | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index dbbad6549..c0f3d23ac 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -435,8 +435,11 @@ public class AddrSvcController { if (json.containsKey("result")) { //我们的结果 JSONObject result = json.getJSONObject("result"); - //我们省级及以上有开发区,直接用我们的结果 - if (result.containsKey("devDistrictName")) { + // 如果是无效信息,则用我们的 + if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) { + basicModel.setResult(result); + }//我们省级及以上有开发区,直接用我们的结果 + else if (result.containsKey("devDistrictName")) { basicModel.setResult(result); } else { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java index 4edaec18d..06d3ef504 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -95,10 +95,12 @@ public class GdBankController { } else if (("否").equals(result.get("countyLevelCity"))) { result.put("prefectureLevelCity", "是"); } - //我们省级及以上有开发区,直接用我们的结果 - if (result.containsKey("devDistrictName")) { + // 如果是无效信息,则用我们的 + if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) { + basicModel.setResult(result); + //我们省级及以上有开发区,直接用我们的结果 + } else if (result.containsKey("devDistrictName")) { basicModel.setResult(result); - } else { try { //如果是江苏省或者浙江省 调用齐梁的结果 @@ -128,7 +130,6 @@ public class GdBankController { map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } - } else { basicModel.setResult(result); } From 16077f376197df629fb409e70615dea1b623a253 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 May 2024 22:24:04 +0800 Subject: [PATCH 6/9] update --- .../ccbevaluation/CcbCommonUtils.groovy | 6 +- .../fangjia/script/ccbevaluation/Test.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 19 + .../script/fuzhou/FzDistrictMatch.groovy | 4 +- .../script/gj/data_handle/DevProcess.groovy | 104 +-- .../data_handle/DistrictAliasProcess.groovy | 2 +- .../gj/data_handle/TownMiddleProcess.groovy | 49 ++ .../gj/district/FangjiaClientUtils.groovy | 4 +- .../FjwBigdataDealBatch.groovy | 141 ++-- .../FjwGuiDingPriceBatch.groovy | 226 ++++++ .../fjwPriceBaseData/FjwPgsqlCommUtils.groovy | 14 + .../FjwThirdPartyPriceBatch.groovy | 674 ++++++++++++++++++ .../fangjia/script/gj/poc/AddPriceTest.groovy | 31 +- 13 files changed, 1119 insertions(+), 157 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 3e18b6b7f..83c9ec794 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -678,11 +678,7 @@ class CcbCommonUtils { } public static def deleteGuidedPrice(def dealdate){ - def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) - - - def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'" + def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'" pgsqlInstance.executeUpdate(sql2) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 5234f7fd4..c42ab1349 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> if(d['J']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index fde3609ec..ea417b329 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1890,6 +1890,25 @@ class CommonDatautils { def dbo = null; while(cur.hasNext()) { dbo = cur.next() + if(dbo.value_type == 1){ + dbo.cal_type = "指导价" + }else if(dbo.value_type == 2){ + dbo.cal_type = "自身样本" + }else if(dbo.value_type == 3){ + dbo.cal_type = "第三方" + }else if(dbo.value_type == 4){ + dbo.cal_type = "交易数据" + }else if(dbo.value_type == 5){ + dbo.cal_type = "周边楼盘" + }else if(dbo.value_type == 6){ + dbo.cal_type = "推导" + }else if(dbo.value_type == 9){ + dbo.cal_type = "神经网络" + }else if(dbo.value_type == 10){ + dbo.cal_type = "无方式" + }else{ + dbo.cal_type = "" + } } return dbo; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index 2b96edc91..d4e07eab4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -22,8 +22,8 @@ class FzDistrictMatch { // task.addPrice() // task.addThirdPrice() -// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") - task.longyanPrice() + task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") +// task.longyanPrice() } def longyanPrice(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index f24f72702..420fa5c2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,7 +29,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.addFlag() // task.matchDev() @@ -42,7 +42,8 @@ class DevProcess { // task.addNewDev() // task.createDevMapping() // task.deleteAddr() - task.synchDevTo43() +// task.synchDevTo43() + } def parseCityAndDist(){ @@ -159,54 +160,69 @@ class DevProcess { } def devSplitRoad(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> - /*if(d['C'] && !d['V']){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['V'] = jsonObject.get("province") - } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['W'] = jsonObject.get("city") - } + def fileName = "/home/goojia/第二批公司.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\第二批公司.xlsx" + } - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['X'] = jsonObject.get("district") - } + def i = 5; + while(i<13){ + ExcelUtils.multiThreadingModify(fileName,{d-> + /*if(d['C'] && !d['V']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['V'] = jsonObject.get("province") + } - if(jsonObject && street && streetNumber && street!="null"){ - d['Y'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['Z'] = jsonObject.get("streetNumber") + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['W'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['X'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['Y'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['Z'] = jsonObject.get("streetNumber") + } + } + }*/ + if(d['C'] && d['B']!='河北省'){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + + if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ + d['O'] = jsonObject.get("dev") } } - }*/ - if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['J'] = jsonObject.get("province") - } + },i) + i++ + } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['K'] = jsonObject.get("city") - } - - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['L'] = jsonObject.get("district") - } - - if(jsonObject && street && streetNumber && street!="null"){ - d['M'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['N'] = jsonObject.get("streetNumber") - } - } - } - },0) } def addFlag(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy index 289e6f68f..c987e3b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy @@ -14,7 +14,7 @@ class DistrictAliasProcess { } def run(){ - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d-> + ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d-> if(d['H']=="是"){ def udata = new BasicDBObject() udata.put("status", 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index ea2bd7f20..32e76368d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -10,6 +10,7 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.client.AggregateIterable import org.bson.Document +import org.bson.types.ObjectId /** * 乡镇街道中间表 @@ -35,6 +36,7 @@ class TownMiddleProcess { static main(args) { def task = new TownMiddleProcess(); task.run() + } def run(){ //创建省份 @@ -52,6 +54,13 @@ class TownMiddleProcess { //添加地级市标志 // addCityFlag() // addPinyin() + //打标记,将开发区与民政部相同的打上1 +// addTjjDevFlag() + //打标记,用于标识乡镇街道是否输出及格式化输出街道 +// addStreetFlag() + +// addDistrictDiffFlag() + } @@ -751,4 +760,44 @@ class TownMiddleProcess { } } + + def addTjjDevFlag(){ + def q = new BasicDBObject("type", "区域") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){ + def udata = new BasicDBObject() + udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1 + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + } + + def addStreetFlag(){ + ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row-> + if(row['I']){ + if(row['I']=="标0"){ + def udata = new BasicDBObject() + udata.put("street_output", 0) //0:不输出 1:输出 + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + }else if(row['I']=="标1"){ + def udata = new BasicDBObject() + udata.put("street_output", 1) //0:不输出 1:输出 + if(row['G']){ + udata.put("street", row['G']) //街道中去除区域 + } + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + } + } + }); + } + + def addDistrictDiffFlag(){ + ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row-> + def flag = row['F'].toString().toDouble().toInteger() + def udata = new BasicDBObject() + udata.put("district_diff", flag) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + }); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 0afe56328..71132614c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -354,13 +354,13 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" + def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) - println jsonObj +// println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy index ebb2ccf1f..21bb3652d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -48,11 +48,9 @@ class FjwBigdataDealBatch extends BaseBatch{ static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", - "postgres", "postgres", "org.postgresql.Driver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { runBatch(new FjwBigdataDealBatch(), args); @@ -61,59 +59,50 @@ class FjwBigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args) + +// println "curDate:"+curDate +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } + + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) + println "历史id加载量:"+dealIds.size() + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } - println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ - addDealPriceMatch(city); } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 -// for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) -// println "历史id加载量:"+dealIds.size() -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -//// delPgData() -// saveToPgSql(rows) -// -//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") -//// saveToPgSql2(datas) + saveToPgSql(rows) + +// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +// saveToPgSql2(datas) } def readTxt(def fileName){ @@ -186,19 +175,7 @@ class FjwBigdataDealBatch extends BaseBatch{ } - def getCommunityPrices(def city){ - def sql = "select commid, avg_price from community_base_price_gj where year_month=${endDate} and cityname='${city}'"; - def map = [:] - try{ - sqlInstance.eachRow(sql,{ - map.put(it.commid, it.avg_price) - }); - }catch(Exception e){ - getPpeGTE15Datas(city) - } - return map - } def findThirdPartyPrice(def city) { def q = new BasicDBObject() @@ -529,7 +506,7 @@ class FjwBigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap, def ppeDistsMap){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all") println city +" 小区量:"+dists.size() def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() @@ -537,9 +514,13 @@ class FjwBigdataDealBatch extends BaseBatch{ // def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i for(def r : rows) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy new file mode 100644 index 000000000..8e494f407 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy @@ -0,0 +1,226 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import org.bson.types.ObjectId +/** + * 建行模型估值项目 + * 人工指导价同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class FjwGuiDingPriceBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); + + def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + + static main(args) { + runBatch(new FjwGuiDingPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] + + FjwPgsqlCommUtils.deleteGuidedPrice(curDate) + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + //添加自己库中的人工指导价 + def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-01", "2022-04-01") + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + } + saveToPgSql(rows) + } + + def findFangDatas(def districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println " 小区量:"+districts.size() + def result = Collections.synchronizedList(new ArrayList()); + for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20]) + } + } + + println "end: "+new Date() + } + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy index 10c94f0ea..21ecf0551 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -21,5 +21,19 @@ class FjwPgsqlCommUtils { return set } + public static def findDistGovPrice(def city){ + def ids = [] + sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it-> + ids << it.gujia_bds_id + }); + return ids; + } + + + public static def deleteGuidedPrice(def dealdate){ + def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'" + sqlInstance.executeUpdate(sql2) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy new file mode 100644 index 000000000..76a828232 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -0,0 +1,674 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.text.SimpleDateFormat +import java.util.concurrent.ConcurrentHashMap + +/** + * 建行模型估值项目 + * 第三方均价数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwThirdPartyPriceBatch extends BaseBatch { + + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304// + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12", + "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwThirdPartyPriceBatch(), args); + } + + @Override + public void process(Object args) { + + def citys = ["乌鲁木齐"]//CcbCommonUtils.getRunningCitys(args) + def stop = false + def month = -1 + while(!stop){ + + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + println weekend.toString() +" "+sDate + runDatas(citys) + month = month -1 + sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM")); + if(sDate == 201912){ + stop = true; + } + } + } + + def runDatas(citys){ + def rows = [] + def index = 0 + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + for(def city : citys){ + index++ + + def gocIdsPrices = [] + if(city == "深圳"){ + gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city) + } + + def statMap = getDistrictStatByCity(city, weekend) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all") + def thirdPriceMap = findThirdPartyPrice(city, statMap) + + def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices) + rows.addAll(rets) + println index+" "+city+" "+rets.size() +" "+rows.size() + } + + saveToPgSql(rows) + } + + + def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def rets = [] + for(def d : dists){ + + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def ret = [] + ret << third.id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.get(d.city) + ret << d.city + ret << data.district_code + ret << data.district + ret << sDate + + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + + } + return rets; + } + + + def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){ + def allRows = [] + for(def d : dists){ + if(gocPrices && gocPrices.contains(d._id.toString())){ + continue; + } + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def dataMap = [:] + dataMap.COMMID = d._id.toString() + dataMap.COMMNAME = d.name + dataMap.CITYID = cityCodeDatas.get(d.city) + dataMap.CITYNAME = d.city + dataMap.DISTID = data.district_code + dataMap.DISTNAME = data.district + dataMap.YEAR_MONTH = sDate + def price_map = [:] + dataMap.price_map = price_map + if(third.zhugezhaofang_esf_price){ + price_map.zhuge_avg_price = third.zhugezhaofang_esf_price + } + if(third.anjuke_esf_price){ + price_map.anjuke_avg_price = third.anjuke_esf_price + } + if(third.soufang_esf_price){ + price_map.fang_avg_price = third.soufang_esf_price + } + if(third.lianjia_esf_price){ + price_map.lianjia_avg_price = third.lianjia_esf_price + } + if(third.cityhouse_esf_price){ + price_map.cityhouse_avg_price = third.cityhouse_esf_price + } + if(third.maitian_esf_price){ + price_map.maitian_avg_price = third.maitian_esf_price + } + if(third.beike_esf_price){ + price_map.ke_avg_price = third.beike_esf_price + } + if(third.get("365house_esf_price")){ + price_map.taofang_avg_price = third.get("365house_esf_price") + } + if(third.zhongyuan_esf_price){ + price_map.zhongyuan_avg_price = third.zhongyuan_esf_price + } + if(third.tuboshi_esf_price){ + price_map.tbs_avg_price = third.tuboshi_esf_price + } + if(third.qfang_esf_price){ + price_map.qfang_avg_price = third.qfang_esf_price + } + if(third.fangtan_esf_price){ + price_map.fangtan_avg_price = third.fangtan_esf_price + } + if(third.fangduoduo_esf_price){ + price_map.fangduoduo_avg_price = third.fangduoduo_esf_price + } + if(third.get("5i5j_esf_price")){ + price_map.wiwj_avg_price = third.get("5i5j_esf_price") + } + dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + allRows << dataMap + } + } + } + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + return results + } + + def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) { + def rets = [] + for (def d : dists) { + + def third = thirdPriceMap.get(d.region + "##" + d.name) + if (third) { + def data = distCodeMap.get(d.region) + if (data) { + third.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << cityCodeDatas.get(d.city) + ret << data.district_code + ret << d._id.toString() + ret << d.name + def web = "" + if(k=="zhugezhaofang_esf_price"){ + web = "诸葛找房" + }else if(k=="anjuke_esf_price"){ + web = "安居客" + }else if(k=="soufang_esf_price"){ + web = "房天下" + }else if(k=="lianjia_esf_price"){ + web = "链家" + }else if(k=="cityhouse_esf_price"){ + web = "禧泰数据" + }else if(k=="maitian_esf_price"){ + web = "麦田" + }else if(k=="beike_esf_price"){ + web = "贝壳" + }else if(k=="365house_esf_price"){ + web = "淘房365" + }else if(k=="zhongyuan_esf_price"){ + web = "中原" + }else if(k=="tuboshi_esf_price"){ + web = "兔博士" + }else if(k=="qfang_esf_price"){ + ret << "Q房" + }else if(k=="fangtan_esf_price"){ + ret << "房探" + }else if(k=="fangduoduo_esf_price"){ + ret << "房多多" + }else if(k=="5i5j_esf_price"){ + ret << "我爱我家" + }else{ + ret << "" + } + + ret << sourceMap.get(web) + ret << web + ret << sDate + ret << v + + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + } + } + return rets; + } + + def createData3(cityid){ + def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def dataMap = [:] + + dataMap.COMMID = r.COMMID + dataMap.COMMNAME = r.COMMNAME + dataMap.CITYID = r.CITYID + dataMap.CITYNAME = r.CITYNAME + dataMap.DISTID = r.DISTID + dataMap.DISTNAME = r.DISTNAME + dataMap.YEAR_MONTH = r.YEAR_MONTH + def price_map = [:] + dataMap.price_map = price_map + if(r.ZHUGE_AVG_PRICE){ + price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE + } + + if(r.ANJUKE_AVG_PRICE){ + price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE + } + + if(r.FANG_AVG_PRICE){ + price_map.fang_avg_price = r.FANG_AVG_PRICE + } + + if(r.LIANJIA_AVG_PRICE){ + price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE + } + + if(r.CITYHOUSE_AVG_PRICE){ + price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE + } + + if(r.MAITIAN_AVG_PRICE){ + price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE + } + + if(r.KE_AVG_PRICE){ + price_map.ke_avg_price = r.KE_AVG_PRICE + } + + if(r.TAOFANG_AVG_PRICE){ + price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE + } + + if(r.ZHONGYUAN_AVG_PRICE){ + price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE + } + + if(r.TBS_AVG_PRICE){ + price_map.tbs_avg_price = r.TBS_AVG_PRICE + } + + if(r.QFANG_AVG_PRICE){ + price_map.qfang_avg_price = r.QFANG_AVG_PRICE + } + + if(r.FANGTAN_AVG_PRICE){ + price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE + } + + if(r.FANGDUODUO_AVG_PRICE){ + price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE + } + + if(r.WIWJ_AVG_PRICE){ + price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE + } + dataMap.updatetime = r.UPDATETIME + + allRows << dataMap + }); + + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + + return results + + } + + + def findThirdPartyPrice(def city, def statMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + dataMap.put("id", db._id.toString()) + map.put(key, dataMap) + } + if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){ + dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){ + dataMap.put("soufang_esf_price", db.soufang_esf_price) + } + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + + if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){ + dataMap.put("365house_esf_price", db.get("365house_esf_price")) + } + + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + + if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){ + dataMap.put("qfang_esf_price", db.qfang_esf_price) + } + + if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){ + dataMap.put("fangtan_esf_price", db.fangtan_esf_price) + } + + if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){ + dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) + } + + if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + def resultMap = [:] + map.each {key, value-> + if(value && value.size()>1){ + if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){ + def price = statMap.get(key) + if(price){ + def priceMap = [:] + def startPrice = (int)(price - price*0.3) + def endPrice = (int)(price*0.3 + price) + value.each{pk,pv-> + if(pk.contains("price")){ + if(pv>=startPrice && pv<=endPrice){ + priceMap.put(pk,pv) + } + }else{ + priceMap.put(pk,pv) + } + } + if(priceMap.size()>1){ + resultMap.put(key, priceMap) + } + } + }else{ + resultMap.put(key, value) + } + + } + } + return resultMap + } + + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + return map + } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(50000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) + } + } + + println "end: "+new Date() + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 81977584c..29214f73f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx" + def filePath = "D:\\数据\\建行\\对标补充数据.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) -// addThirdPrice(filePath) - addFjwPrice(filePath) + addThirdPrice(filePath) +// addFjwPrice(filePath) // exportPkPrice() // addRegion() @@ -190,11 +190,11 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['M']) { - def dist = CommonDatautils.findDistrictById(d['M']) + if (d['G']) { + def dist = CommonDatautils.findDistrictById(d['G']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) - d['AN'] = thirdPrice + d['Z'] = thirdPrice } } }) @@ -206,7 +206,7 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302)) + q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null @@ -216,10 +216,10 @@ class AddPriceTest { break; } - if(!price && db.beike_esf_price && db.beike_esf_price>2000){ - price = db.beike_esf_price - break - } +// if(!price && db.beike_esf_price && db.beike_esf_price>2000){ +// price = db.beike_esf_price +// break +// } } return price } @@ -256,14 +256,15 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['F']){ - if(d['E'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['E']) + if(!d['W']){ + if(d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) if(dist){ // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ - d['F'] = dbo.avg_price + d['W'] = dbo.avg_price + d['X'] = dbo.cal_type // d['J'] = dbo.avg_price_2b } } From 8c1a656c40f62dbd2c32a19300289db73107f3aa Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 May 2024 22:38:09 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E6=94=B9=E6=96=87=E4=BB=B6=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 420fa5c2d..c13fc17ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -161,7 +161,7 @@ class DevProcess { def devSplitRoad(){ - def fileName = "/home/goojia/第二批公司.xlsx" + def fileName = "/home/goojia/gj/company.xlsx" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\数据建设\\第二批公司.xlsx" } From 192ea61ea7688fdbcf5b55af8bb828b6a716cb42 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 May 2024 22:41:48 +0800 Subject: [PATCH 8/9] =?UTF-8?q?5-=E3=80=8B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index c13fc17ee..82eee3f4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -166,7 +166,7 @@ class DevProcess { fileName = "D:\\数据\\数据建设\\第二批公司.xlsx" } - def i = 5; + def i = 0; while(i<13){ ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ From 2f861a6521e5513fd21a16b4b778611cb325dddf Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 May 2024 23:55:38 +0800 Subject: [PATCH 9/9] update --- .../script/gj/data_handle/DevProcess.groovy | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 82eee3f4a..374ad79bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -160,14 +160,12 @@ class DevProcess { } def devSplitRoad(){ - - def fileName = "/home/goojia/gj/company.xlsx" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\第二批公司.xlsx" - } - - def i = 0; + def i = 1; while(i<13){ + def fileName = "/home/goojia/gj/company-${i}.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\第二批公司-${i}.xlsx" + } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) @@ -219,7 +217,7 @@ class DevProcess { d['O'] = jsonObject.get("dev") } } - },i) + },0) i++ }