diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java new file mode 100644 index 000000000..845358f5b --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -0,0 +1,71 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.HttpRequest; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; +import com.fangjia.open.model.AddressModel; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * 光大接口 + */ +@Authority(role = Role.USER) +public class GdBankController { + + /** + * 涉农识别-新规 + * + * @param model + * @return + */ + public Object villageIdentifyAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); + params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null); + params.put("redis", (model.getRadius() != null && model.getRadius() != 0) ? model.getRadius().toString() : null); + params.put("address", model.getAddress()); + params.put("showFullValue", "2"); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8"); + } catch (Exception e) { + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + if (json != null) { + int code = json.getInt("msgCode"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if (json.containsKey("result")) { + JSONObject result = json.getJSONObject("result"); + if (("是").equals(result.get("countyLevelCity"))){ + result.put("prefectureLevelCity", "否"); + } else if(("否").equals(result.get("countyLevelCity"))){ + result.put("prefectureLevelCity", "是"); + } + basicModel.setResult(result); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } +} diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl index 5160cbd1a..d9b670394 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl @@ -133,7 +133,7 @@
浜ц瘉寤虹瓚闈㈢Н锛�
-
diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index e04dc02ff..b98b25ab1 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -537,21 +537,21 @@ - - com.aliyun - aliyun-java-sdk-address-purification - 1.0.1 - - - com.aliyun - aliyun-java-sdk-core - 4.5.9 - - - com.alibaba - fastjson - 1.2.83 - + + + + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index dabf9a38b..40fe526e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -2,38 +2,36 @@ package com.fangjia.script public enum Watcher { //离职同事 - LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'), - YanZhou('13621626685','duxiaoyu@fangjia.com'), - HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'), - SongYingXia('13621626685', 'duxiaoyu@fangjia.com'), - YuKai('13621626685', 'duxiaoyu@fangjia.com'), - RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'), - Reda('13621626685', 'duxiaoyu@fangjia.com'), - Tomzhao('13621626685','duxiaoyu@fangjia.com'), - YuHai('13621626685','duxiaoyu@fangjia.com'), - Shibb('13621626685','duxiaoyu@fangjia.com'), - HeLvYuan('13621626685','duxiaoyu@fangjia.com'), - WangYuHui('13621626685','duxiaoyu@fangjia.com'), - LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'), - FangRui('17717680589','fangrui@fangjia.com'), - //DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), + LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'), + YanZhou('13621626685','wangwenyan@fangjia.com'), + HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'), + SongYingXia('13621626685', 'wangwenyan@fangjia.com'), + YuKai('13621626685', 'wangwenyan@fangjia.com'), + RuanYaDong('13621626685', 'wangwenyan@fangjia.com'), + Reda('13621626685', 'wangwenyan@fangjia.com'), + Tomzhao('13621626685','wangwenyan@fangjia.com'), + YuHai('13621626685','wangwenyan@fangjia.com'), + Shibb('13621626685','wangwenyan@fangjia.com'), + HeLvYuan('13621626685','wangwenyan@fangjia.com'), + WangYuHui('13621626685','wangwenyan@fangjia.com'), + LinXiaoShuai('15801810393','wangwenyan@fangjia.com'), + FangRui('17717680589','wangwenyan@fangjia.com'), + DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'), //在职同事 - DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'), - LiXue('13311649646','lixue@fangjia.com'), + LiXue('13311649646','wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'), GaoJuan('15021384995','gaojuan@fangjia.com'), - Hubingliang('15179856208','hubingliang@126.com'), - ZhouTongYun('13585834921','zhoutongyun@fangjia.com'), - YinJiHuan('18217642009','yinjihuan@fangjia.com'), - Xuhe('13862416422','xuhe@fangjia.com'), - ZongZhiMin('18260046486','zongzhimin@fangjia.com'), - RaoYanHui('18516539752','raoyanhui@fangjia.com'), - BaYi('15300424133','bayi@fangjia.com'), - WeiLei('18101751850','weilei@fangjia.com'), - JiangYaHui('13684913780','jiangyahui@fangjia.com'), - ShenJian('13764369683', 'shenjian@fangjia.com'), - MaJinTao('13607600062','majintao@fangjia.com') + Hubingliang('15179856208','wangwenyan@fangjia.com'), + ZhouTongYun('13585834921','wangwenyan@fangjia.com'), + YinJiHuan('18217642009','wangwenyan@fangjia.com'), + Xuhe('13862416422','wangwenyan@fangjia.com'), + ZongZhiMin('18260046486','wangwenyan@fangjia.com'), + RaoYanHui('18516539752','wangwenyan@fangjia.com'), + BaYi('15300424133','wangwenyan@fangjia.com'), + WeiLei('18101751850','wangwenyan@fangjia.com'), + JiangYaHui('13684913780','wangwenyan@fangjia.com'), + ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy index 9e0bfffd0..796f9b27b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy @@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{ println(content) sendEmail(Watcher.WangWenYan.email, "后台统计量", content) sendEmail("wangchencen@fangjia.com", "后台统计量", content) - sendEmail("caoyuyan@fangjia.com", "后台统计量", content) - sendEmail("shenyuan@fangjia.com", "后台统计量", content) - sendEmail("xuminxia@fangjia.com", "后台统计量", content) - sendEmail("huangcunliang@fangjia.com", "后台统计量", content) - sendEmail("yaojingmeng@fangjia.com", "后台统计量", content) - sendEmail("muliangshu@fangjia.com", "后台统计量", content) - sendEmail("sunjian@fangjia.com", "后台统计量", content) +// sendEmail("caoyuyan@fangjia.com", "后台统计量", content) +// sendEmail("shenyuan@fangjia.com", "后台统计量", content) +// sendEmail("xuminxia@fangjia.com", "后台统计量", content) +// sendEmail("huangcunliang@fangjia.com", "后台统计量", content) +// sendEmail("yaojingmeng@fangjia.com", "后台统计量", content) +// sendEmail("muliangshu@fangjia.com", "后台统计量", content) +// sendEmail("sunjian@fangjia.com", "后台统计量", content) } } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy index 41a7088f2..2f785e37c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy @@ -29,7 +29,7 @@ class HouseChangingMonitorBatch extends BaseBatch { DBCollection COLLECTION = MongoUtils.getCollectionByName("fangjia_log", "house_changing_monitor", "log.mongo.db.fangjia.com") Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); def EMAILS = [ - 'duxiaoyu@fangjia.com' + 'gaojuan@fangjia.com' ] public static void main(String[] args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index dc3ee5d27..9be5b8c21 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -21,6 +21,7 @@ import org.bson.types.ObjectId /** * 通知计算二手房均价 + * 发送平米租金命令 * @author duxiaoyu * */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy index a1cf124eb..fdf9f1e5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy @@ -39,7 +39,7 @@ class WebAverageLabor extends BaseBatch{ public static final def CITIES = ["上海", "成都", "北京", "广州", "深圳"] - public static final def EMAILS = ["duxiaoyu@fangjia.com"] + public static final def EMAILS = ["wangwenyan@fangjia.com"] static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy index a69810dab..ea0274d5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy @@ -24,15 +24,16 @@ class BatchAddPersonGuiPrice { def run(){ - def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx" + def path = "D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx" def datas = ExcelUtils.read(path, 0, 1) for(def d : datas){ - def row = findGuidingPrice(d['B'], d['C'], d['D']) + def row = findGuidingPrice(d['B'], d['C'], d['C']) if(!row){ - def obj = CommonDatautils.findDistrictById(d['A']) - save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2) +// def obj = CommonDatautils.findDistrictById(d['A']) + def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + save(d['A'], d['B'], d['C'], d['Q'], obj.estate_type2, d['P']) //发重算均价命令 - addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date()) + addPrice(d['A'], d['B'], d['C'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),new Date()) } } } @@ -43,13 +44,13 @@ class BatchAddPersonGuiPrice { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("guiding_person" , "付贵生") - q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd"))) +// q.put("guiding_person" , person) + q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-03-20", "yyyy-MM-dd"))) def dbo = GUIDING_PRICE_COLL.findOne(q) return dbo } - def save(city, region, name, price, estate_type){ + def save(city, region, name, price, estate_type, person){ DBObject obj = new BasicDBObject(); obj.put("city", city); obj.put("region", region); @@ -62,7 +63,7 @@ class BatchAddPersonGuiPrice { obj.put("m_date", new Date()); obj.put("estate_type" , estate_type); obj.put("house_type" , 0) - obj.put("guiding_person" , "陈杰") + obj.put("guiding_person" , person) GUIDING_PRICE_COLL.save(obj); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 8bc6aff7c..fca28ce3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -387,26 +387,43 @@ class DealPriceDataProcess { } def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys_all +// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" + def citys = ["469005"]//CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() - for(def city : citys) { - def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 - def total = 0 - println city +" "+rets.size() + for(def cityid : citys) { + def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 + println cityid +" "+rets.size() if(rets){ - set.add(city) + set.add(cityid) } for(def m : rets){ - def cityId = cityCodeDatas.get(city) +// def cityId = cityCodeDatas.get(city) // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // sqlInstance236.executeUpdate(sql) + def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + def total = 0 + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println cityid +" "+m.commid+" (按价格)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + pgsqlInstance.executeUpdate(sql) + }else{ - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - pgsqlInstance.executeUpdate(sql) - + querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println cityId +" "+m.commid+" (按小区)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.executeUpdate(sql) + } + } } } println set.toString() @@ -432,8 +449,8 @@ class DealPriceDataProcess { return rets } - def findPeGteThirdParty2(cityName, yearmonth){ - def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>20" + def findPeGteThirdParty2(cityid, yearmonth){ + def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20" def rets = [] pgsqlInstance.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index 2fd7e0c2f..cfbd78a2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -27,4 +27,13 @@ class PgsqlCommUtils { }); return count } + + public static def findDistrictMap(){ + def sql = "select urbn_id,gujia_admndiv_id,gujia_admndiv_nm from base.t0831_bigdata_district_gj_h" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.put(r.urbn_id+"#"+r.gujia_admndiv_id, r.gujia_admndiv_nm) + }); + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 3a55f12b6..50cc6343f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -69,6 +69,12 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCityPrice(queryDate) exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) + // 新增两张数据表 + // avm_evaluate_rate_city_gj_h + // avm_evaluate_rate_comm_gj_h + exportEvaluateRateCity(queryDate) + exportEvaluateRateComm(queryDate) + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 // exportCommunityBasePrice(lastMonth) // exportCommunityPdescribeGj(lastMonth) @@ -228,6 +234,79 @@ class ExportDistrictRongHe extends BaseBatch{ CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "GUIDED_PRICE_GJ 上传Ftp完成......" } + // avm_evaluate_rate_city_gj_h gujia_clc_yrmo_val =202402 and gujia_clc_yrmo_val<=202403 gujia_clc_yrmo_val>=to_date('${startDateStr}','yyyy-MM-dd') and gujia_clc_yrmo_val<=to_date('${endDateStr}','yyyy-MM-dd') + def exportEvaluateRateCity(queryDate){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bld_tp_val + data << r.gujia_bld_tp_val_coef + data << r.gujia_clc_yrmo_val + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + // FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_ + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat" + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + + println "EVALUATE_RATE_CITY_FJW_ADD 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "EVALUATE_RATE_CITY_FJW_ADD 上传Ftp完成......" + } + + def exportEvaluateRateComm(queryDate){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' "; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_clc_yrmo_val + data << r.basic_price + data << r.area_avg + data << r.unitprc_avg + data << r.comm_type + data << r.area_avg_coef + data << r.basic_price_coef + data << r.buildarea_coef + data << r.type_coef + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + // FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat" + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + + println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 上传Ftp完成......" + } def exportBigdataCommbuildstruGJ(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ad59e008c..e0e58814f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -40,6 +40,8 @@ import java.util.zip.GZIPInputStream class CommonDatautils { + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -1175,6 +1177,28 @@ class CommonDatautils { return null; } + + + public static def findAllStatusDistrict(String city,String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("region", region); + q.put("name", name); + q.put("cat", cat); + + def rets = [] + def cursor = SEAWEED_MONGO_COLL.find(q); + if(cursor){ + for(def obj : cursor){ + rets << obj; + } + + } + return rets; + } + + public static def findDistrictsByName(String city, String name, String cat){ DBObject q = new BasicDBObject(); @@ -1537,6 +1561,32 @@ class CommonDatautils { } + static def findAllStatusDistrictByAlias(def city, def region, def name, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + + def names = []; + names << name; + BasicDBObject query = new BasicDBObject(); + query.putAt(QueryOperators.IN, names); + q.put("alias", query); + + + def dsCursor = getDataStoreConnection().find(q); + def ret = []; + for(def obj : dsCursor){ + ret << obj; + } + + return ret; + } + + static def findDistrictByAddress(def city, def region, def address, def cat){ DBObject q = new BasicDBObject(); if(region){ @@ -3764,6 +3814,135 @@ class CommonDatautils { return rets; } + + + static def findHouses2(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range) + + if(region){ + queryBuilder.must(QueryBuilders.termQuery("region", region)) + } + + if(name || alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + def avgPrices = [] + def areas = [] + def totalPrices = [] + + if(houses){ + for (int k=0;k1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } + avgPrices << seawater.avg_price + areas << seawater.area + totalPrices << seawater.total_price + } + } + + def retMap = [:] + if(areas){ + retMap.avg_price = avgPrices + retMap.area = areas + retMap.total_price = totalPrices + } + return retMap; + } + + /** + * 是否是别墅 + * @return + */ + public static boolean isVilla(Double area, String name, Integer height, Integer room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + }else if(height != null){ + if (height <= 3) { + if (area > 200){ + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + static def parseFloor(def floorDesc){ if(floorDesc){ floorDesc = floorDesc.replaceAll("[\\s ]+", "") @@ -3919,6 +4098,29 @@ class CommonDatautils { return avg_price } + static def findMatchTrades(city, region, name, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("match_region", region) + q.put("match_name", name) + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + /** @@ -3993,6 +4195,32 @@ class CommonDatautils { return total; } + + static def findLogs(user_id, requestUri, startDate, endDate){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", user_id)).must(QueryBuilders.termQuery("request_uri", requestUri)) + if(startDate){ + def startTime = DateUtils.parseFromTightYYYYMMDD(startDate) + def endTime = DateUtils.parseFromTightYYYYMMDD(endDate) + def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false); + queryBuilder.must(date_range_cdate); + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + if(houses){ + for (int k=0;k3000){ dataMap.put("beike_esf_price", db.beike_esf_price) } + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + if(!dataMap.containsKey("xingfuli_esf_price") && db.xingfuli_esf_price && db.xingfuli_esf_price>3000){ + dataMap.put("xingfuli_esf_price", db.xingfuli_esf_price) + } + + if(!dataMap.containsKey("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } } return map } @@ -4337,6 +4582,23 @@ class CommonDatautils { } + static def findHouseAvgPrice(city, region , start, end){ + def prices = [] + def sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date > ${start} and p_date<${end} and (root='lianjia.com' or root='ke.com')" + if(region){ + sql = sql + " and region='${region}'" + } + ElasticUtils.executeSqlQuery(sql, { + prices << it.avg_price + }) + if(prices && prices.size()>3){ + def avg_price = (int)(prices.sum()/prices.size()) + println region + " "+ prices.size() +" "+avg_price +" "+start+" "+end + return avg_price + } + return null + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy index 1f060779c..083b4973d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy @@ -24,16 +24,16 @@ class DistrictDataCoverBatch extends BaseBatch { public DSCollection dsCollection = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "district_facilitys", "mongo.db.fangjia.com") public static final def EMAILS =[ - "duxiaoyu@fangjia.com", - "ruanyadong@fangjia.com", - "xiamin@fangjia.com", - "zhutianli@fangjia.com", - "joe@fangjia.com", - "yukai@fangjia.com", - "jiangxiaochun@fangjia.com", +// "duxiaoyu@fangjia.com", +// "ruanyadong@fangjia.com", +// "xiamin@fangjia.com", +// "zhutianli@fangjia.com", +// "joe@fangjia.com", +// "yukai@fangjia.com", +// "jiangxiaochun@fangjia.com", "gaojuan@fangjia.com", - "liuxingwang@fangjia.com", - "xiaohai@fangjia.com", +// "liuxingwang@fangjia.com", +// "xiaohai@fangjia.com", ] static def FIELDS_MAP = [ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy index 84bb4a078..9b187a3f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy @@ -33,10 +33,7 @@ class DistrictFieldCoverBatch extends BaseBatch { final static String coverDomain = "mongo.db.fangjia.com" final static String datastoreDomain = "seaweed.db.fangjia.com" final static def emails = [ - "duxiaoyu@fangjia.com", - "joe@fangjia.com", "yukai@fangjia.com", "yanzhou@fangjia.com", - "gaojuan@fangjia.com", "yuanyadong@fangjia.com", "liuxingwang@fangjia.com", - "xiamin@fangjia.com", "zhutianli@fangjia.com" + "gaojuan@fangjia.com" ] // final static def emails = [ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy index 74ec3b25f..63fbf843d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy @@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch { static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com") public static final def EMAILS = [ - "duxiaoyu@fangjia.com", +// "duxiaoyu@fangjia.com", // "ruanyadong@fangjia.com", // "xiamin@fangjia.com", // "zhutianli@fangjia.com", - "joe@fangjia.com", - "yukai@fangjia.com", - "jiangxiaochun@fangjia.com", - "gaojuan@fangjia.com", - "shijing@fangjia.com", - "xiaohai@fangjia.com", +// "joe@fangjia.com", +// "yukai@fangjia.com", +// "jiangxiaochun@fangjia.com", + "gaojuan@fangjia.com" +// "shijing@fangjia.com", +// "xiaohai@fangjia.com", ] static def FIELDS_MAP = [ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy index 6212900b5..a0ed5c281 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy @@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') def static emails = [ - 'xuqi@fangjia.com', - 'zongzhimin@fangjia.com' + 'wangwenyan@fangjia.com', + 'gaojuan@fangjia.com' ] static void main(String[] args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy index fa3d57a71..5542b2c16 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.district +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.MongoUtils import com.mongodb.DBCursor +import org.bson.types.ObjectId import java.util.List @@ -40,6 +42,24 @@ class DistrictHouseBatch extends BaseBatch { @Override public void process(Object args) { + +// DBObject q = new BasicDBObject(); +// q.put("cat", "sell"); +// q.put("city","常州") +// q.put("district_name","清河苑一区") +// def dbcursor = CommonDatautils.getHouseDataStoreConnection().find(q) +// for(def obj : dbcursor){ +// obj.put("c_date", new Date()) +// CommonDatautils.getHouseDataStoreConnection().save(obj) +// } + + +// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","清河苑一区_溧阳_常州")).each{ +// println(it) +// } + + + def cityList = cityList(); int c = 0 @@ -47,18 +67,38 @@ class DistrictHouseBatch extends BaseBatch { c++ try { long sn = Long.parseLong("${new Date().format('yyyyMMddHHmmssSSS')}") - List districts = findDistrictByCity(city) +// List districts = findDistrictByCity(city) + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"]) Map sellCountMap = facetDistrictInHouse("sell", city); Map rentCountMap = facetDistrictInHouse("rent", city); println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) for(DBObject district : districts) { - String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city") - long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 - long rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 + def names = [] + names << district.name + if(district.alias && districts.alias.size()>0){ + for(def name : district.alias){ + if(!names.contains(name)){ + names << name + } + } + } + + long sellCount = 0 + long rentCount = 0 + for(def name : names){ + String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city") + if(!sellCount){ + sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 + } + if(!rentCount){ + rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 + } + } saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) } HOUSE_COLL.remove(new BasicDBObject("city", city).append('sn', new BasicDBObject('$ne', sn))) + println c + " " + city }catch (Exception e) { e.printStackTrace(); } @@ -75,6 +115,7 @@ class DistrictHouseBatch extends BaseBatch { sellQuery.put('$facet', facet) sellQuery.put('city', city) sellQuery.put('cat',cat) +// sellQuery.put("district_name","新城帝景南区") // sellQuery.put('visible',0) DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0) List sellFacetResult = sellDSCursor.getFacetResults(); @@ -109,13 +150,14 @@ class DistrictHouseBatch extends BaseBatch { fields.put("city", 1) fields.put("region", 1) fields.put("name", 1) - + fields.put("alias", 1) def ret = [] DBObject q = new BasicDBObject() q.put('city', city) q.put('cat', 'district') q.put('visible', 0) + q.put("name","清河苑") DSCursor cur = SEAWEED.find(q, fields).batchSize(2000) Iterator iter = cur.iterator() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy index 46195928b..9c1578e2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy @@ -29,9 +29,7 @@ class EvaluationFieldProcent extends BaseBatch{ public static final DBCollection EVALUA_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") public static final DBCollection EVALUA_COVER_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation_field_coverage", "mongo.db.fangjia.com") - final static def emails = ["xiaohai@fangjia.com","jiangxiaochun@fangjia.com","limingmin@fangjia.com","duxiaoyu@fangjia.com", - "joe@fangjia.com", "yukai@fangjia.com", "gaojuan@fangjia.com", "yuanyadong@fangjia.com", - "xiamin@fangjia.com", "zhutianli@fangjia.com"] + final static def emails = [ "gaojuan@fangjia.com"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy index a1043fb29..5091edaa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy @@ -30,7 +30,7 @@ class FjtAnalyCrawlerIp extends BaseBatch{ public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final DBCollection addressMatchLogColl = MongoUtils.getCollectionByName2("fangjia", "address_match_log", lineUrl) - final static def emails = ["gaojuan@fangjia.com","jiangxiaochun@fangjia.com","duxiaoyu@fangjia.com","wangwenyan@fangjia.com","luxuping@fangjia.com","yaojiayi@fangjia.com"] + final static def emails = ["gaojuan@fangjia.com"] static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy index ff8e2fb8a..29478337f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy @@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch { href = "点击下载附件" jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href) jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href) - jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href) +// jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href) } public static Map getLngLatByAddress(String city, String address){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 0b6a136ec..beeafcb09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -74,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch { def day = new Date().getDate() - def startMDate = DateUtils.truncateDay(new Date()-1) + def startMDate = DateUtils.truncateDay(new Date()-7) def endMDate = DateUtils.truncateDay(new Date()) def index = 0 @@ -144,7 +145,8 @@ class DistrictSellStatBatch extends BaseBatch { void run() { try { totalCount.addAndGet(1) - def key = dist.region +"##" + dist.name + def type = dist.estate_type2=="别墅"?"别墅":"小区" + def key = dist.region +"##" + dist.name+"##"+ type def weekendPriceObj = weekendPriceMap.get(key) if(weekendPriceObj){ @@ -153,8 +155,13 @@ class DistrictSellStatBatch extends BaseBatch { def lastYearPriceObj = lastYearPriceMap.get(key) double mom = 0d - if(lastMonthPriceObj) { - mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) + if(lastMonthPriceObj && lastMonthPriceObj.avg_price>0) { + try{ + mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) + }catch(Exception e){ + println key+" 出错................." + } + } def curPrice = weekendPriceObj.avg_price @@ -192,7 +199,7 @@ class DistrictSellStatBatch extends BaseBatch { curPrice = 0 } if(curPriceObj){ - if(curPrice!=weekendPriceObj.avg_price){ + if(curPrice!=curPriceObj.avg_price){ def data = new BasicDBObject() data.put("m_date", new Date()) data.put("commid", dist._id.toString()) @@ -263,7 +270,7 @@ class DistrictSellStatBatch extends BaseBatch { } } if(!priceMap.containsKey(d.region +"##"+d.district_name)){ - priceMap.put(d.region +"##"+d.district_name, d) + priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d) } } return priceMap @@ -355,4 +362,5 @@ class DistrictSellStatBatch extends BaseBatch { // } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 02d267a37..893ad9876 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -9,6 +9,7 @@ import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBCursor import com.mongodb.DBObject +import com.rabbitmq.client.AMQP import com.rabbitmq.client.impl.AMQImpl import org.apache.commons.io.FileUtils import org.bson.types.ObjectId @@ -28,10 +29,11 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() - task.priceMarkupRun(); - task.regionPriceTrendRun() +// task.priceMarkupRun(); +// task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() + task.testNewPrice() } def margeExcel(){ @@ -163,7 +165,7 @@ class FzDataResultStat { def exportRegion(city, months){ - def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"]; def datas =[] def header = ["城市", "区域"] for(def m : months){ @@ -206,8 +208,12 @@ class FzDataResultStat { DBObject dbo = cur.next() prices << dbo.get("avg_price"); } - def avg_price = (int)(prices.sum()/prices.size()) - return avg_price; + if(prices){ + def avg_price = (int)(prices.sum()/prices.size()) + return avg_price; + } + + return null; } @@ -293,4 +299,20 @@ class FzDataResultStat { } return rets } + + def testNewPrice(){ + ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d-> + if(d['A']){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("region",d['B']) + q.put("name",d['C']) + q.put("month", 202402) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if(obj){ + d['S'] = obj.avg_price + } + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index cae471308..2b96edc91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.fuzhou import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -19,7 +20,72 @@ class FzDistrictMatch { def task = new FzDistrictMatch() // task.run(); // task.addPrice() - task.addThirdPrice() +// task.addThirdPrice() + +// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") + task.longyanPrice() + } + + def longyanPrice(){ +// def dists = CommonDatautils.findAllDistricts("龙岩","district", ["city","region","name","estate_type2"]) +// def rets = [] +// rets << ["Id", "城市","区域", "名称", "物业类型", "上个月价格toB","上个月价格","最新价格toB","最新价格"] +// for(def d : dists){ +// def ret = [] +// ret << d._id.toString() +// ret << d.city +// ret << d.region +// ret << d.name +// ret << d.estate_type2 +// def obj2 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-03-10"), 1) +// if(obj2){ +// ret << obj2.avg_price_2b +// ret << obj2.avg_price +// } +// +// def obj1 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-04-07"), 1) +// if(obj1){ +// ret << obj1.avg_price_2b +// ret << obj1.avg_price +// } +// rets << ret +// } +// ExcelUtils.write("D:\\数据\\福州\\龙岩价格对比.xlsx", rets, "龙岩") + + + addThirdPrice("D:\\数据\\福州\\龙岩价格对比.xlsx") + } + + def addThirdPrice2(path){ + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice(row['A'], row['B'], row['C'], 202312, 202402) + def priceMap = cityMap.get(row['A']+"##"+row['B']+"##"+row['C']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['J'] = priceMap.get("lianjia_esf_price") + } + + if(!row['J'] && priceMap.containsKey("beike_esf_price")){ + row['J'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("maitian_esf_price")){ + row['K'] = priceMap.get("maitian_esf_price") + } + } + + def tradeAvgPrice = CommonDatautils.findMatchTrades(row['A'], row['B'], row['C'], "2023-08-01", "2024-01-31"); + if(tradeAvgPrice){ + row['L'] = (int)tradeAvgPrice + } + + def priceObj = CommonDatautils.findDistrictPrice(row['A'], row['B'], row['C']) + if(priceObj){ + row['M'] = priceObj.avg_price_2b + } + + }) } def addThirdPrice(){ @@ -173,4 +239,57 @@ class FzDistrictMatch { } },0) } + + + def addThirdPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['C'] && d['C']=="新罗"){ + def cityMap = CommonDatautils.findThirdPartyPrice(d['B'], d['C'], d['D'], 202312, 202403) + def priceMap = cityMap.get(d['B']+"##"+d['C']+"##"+d['D']) + + if(priceMap && priceMap.zhugezhaofang_esf_price){ + d['J'] = priceMap.zhugezhaofang_esf_price + } + + if(priceMap && priceMap.anjuke_esf_price){ + d['K'] = priceMap.anjuke_esf_price + } + + if(priceMap && priceMap.soufang_esf_price) { + d['L'] = priceMap.soufang_esf_price + } + + if(priceMap && priceMap.lianjia_esf_price){ + d['M'] = priceMap.lianjia_esf_price + } + + if(priceMap && priceMap.cityhouse_esf_price){ + d['N'] = priceMap.cityhouse_esf_price + } + + if(priceMap && priceMap.zhongyuan_esf_price){ + d['O'] = priceMap.zhongyuan_esf_price + } + + if(priceMap && priceMap.xingfuli_esf_price){ + d['P'] = priceMap.xingfuli_esf_price + } + if(priceMap && priceMap.tuboshi_esf_price){ + d['Q'] = priceMap.tuboshi_esf_price + } + + if(priceMap && priceMap.get("5i5j_esf_price")){ + d['R'] = priceMap.get("5i5j_esf_price") + } + + def sellMap = CommonDatautils.findHouses2(d['B'], d['C'], "sell", 20231001, 20240407, d['D'], null, d['E']) + if(sellMap && sellMap.avg_price){ + if(sellMap.avg_price.size()>=3){ + d['S'] = Math.round(sellMap.avg_price.sum() / sellMap.avg_price.size()); + } + } + } + },0) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 9e365230a..ad2cf8430 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 28fb38a92..cb2fcce9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -20,16 +20,16 @@ class AddAvgpriceBatch { public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static { - citys = ["徐州"]//CommonDatautils.findAllCity() + citys = ["宜昌"]//CommonDatautils.findAllCity() println "加载城市完成:"+citys.size() } static main(args) { def task = new AddAvgpriceBatch(); -// task.run1(); + task.run1(); // task.test() // task.sendPriceCommand() - task.sendNewDistPrice() +// task.sendNewDistPrice() } def sendNewDistPrice(){ @@ -107,7 +107,7 @@ class AddAvgpriceBatch { println "------------------------------------------------------------" println "${city}小区量:"+districtKeysMap.size() if(districtKeysMap){ - def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26")) + def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2024-04-07")) println "【2023-03-26】共计均价小区:"+lastWeekendStats.size() //库中小区没有计算过均价的小区 def districtKeys = districtKeysMap.keySet(); @@ -118,7 +118,7 @@ class AddAvgpriceBatch { def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) //发重算均价命令 - addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd')) + addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2024-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2024-04-07','yyyy-MM-dd')) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy index fc989d72f..49b75a911 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.gj +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.poi.entity.Community import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -12,7 +14,41 @@ class CitySellStatTest { static main(args) { def task = new CitySellStatTest(); - task.run() +// task.run() + task.cityPriceRun() + } + + def cityPriceRun(){ + def cityObjs = CommonDatautils.findCitys(); + def rets = [] + rets << ["序号","城市","城市级别","城市均价起始月","起始均价","城市均价最新月","最新均价"] + def index = 1 + for(def o : cityObjs){ + def ret = [] + ret << index + ret << o.display + ret << o.new_city_grade + def row1 = findCityMonthPrice(o.display, 1) + if(row1){ + ret << row1.date + ret << row1.avg_price + }else{ + ret << "" + ret << "" + } + def row2 = findCityMonthPrice(o.display, -1) + if(row2){ + ret << row2.date + ret << row2.avg_price + }else{ + ret << "" + ret << "" + } + rets << ret + index++ + } + + ExcelUtils.write("D:\\数据\\其它\\全国城市均价.xlsx", rets, "城市均价") } def run(){ @@ -25,18 +61,20 @@ class CitySellStatTest { },0) } - def findFirstMonth(def city){ + def findCityMonthPrice(def city, def asc){ def q = new BasicDBObject(); q.put("city", city) - def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("_id",1)).limit(1) - def month; + def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("date", asc)).limit(1) + def obj = null for(def o : dbcursor){ - month = o.date + obj = o } - return month + return obj } + + def findTotalMonth(def city, def month){ def q = new BasicDBObject(); q.put("city", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index e6812a2e4..82893abe6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -23,18 +23,22 @@ class EsSearchUtils { ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString()) def rets = [] ElasticUtils.executeSqlQuery(sql,{ data-> - def m = [:] - m.city = data.cityname - m.region = data.adname - m.name = data.name - m.address = data.address - m.type = data.type - m.code = data.typecode + def mapName = data.name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + def newName = name.replaceAll('[·\\(\\)()]|·', '') + if(newName == mapName){ + def m = [:] + m.city = data.cityname + m.region = data.adname + m.name = data.name + m.address = data.address + m.type = data.type + m.code = data.typecode - def location = data.location - m.lng = location.lon - m.lat = location.lat - rets << m + def location = data.location + m.lng = location.lon + m.lat = location.lat + rets << m + } }) return rets; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 621b548e7..0db9c5bb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,9 +52,9 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo - task.esfParser("D:\\数据\\友达\\2024年1月\\查询1.DBF");//友达二手房存入mongo + task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es -// task.esfhouseToSaveEs();//二手房保存到es + task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy index 3289104fe..7ecf720df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy @@ -31,7 +31,34 @@ class CityStatExport { // task.regionStat() // task.stat() // task.houseStat() - task.addFieldStat() +// task.addFieldStat() + + task.exportRegionAvgPrice() + } + + def exportRegionAvgPrice(){ + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"]; + def add = -9; + def months = [] + while(add<=0){ + months << DateUtils.addMonth(new Date(), add) + add++ + } + def rets = [] + rets << months + for(def region : regions){ + def row = [] + row << region + for(def i=0; i + if(d['B']){ + def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city", "region", "name"]) + d['D'] = districts.size() + } + },0); } def dealPriceStat(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy new file mode 100644 index 000000000..0979849cf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +//郑州银行数据导出 +class ZhengzhouBlankStat { + public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com") + + public static final def DISTRICT_FIELDS_MAP = [ + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' + ] + + static main(args) { + def task = new ZhengzhouBlankStat(); + task.run() + } + + def run(){ + def citys = ["郑州","洛阳","平顶山","安阳","许昌","南阳","新乡","焦作","开封","信阳","驻马店","漯河","鹤壁","周口","濮阳","三门峡","商丘","济源"] + //statRegionDistrictCount(citys) +// statCityProcent(citys) + statCityHouseloudongProcent(citys) + + } + + def statRegionDistrictCount(citys){ + def rets = [] + rets << ["城市", "区县", "楼盘数量"] + for(def c : citys){ + def regions = CommonDatautils.getCityRegions(c) + + for(def r : regions){ + def count = CommonDatautils.findRegionDistrictCount("district", c, r) + def ret = [] + ret << c + ret << r + ret << count + rets << ret + } + + } + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省区县小区量统计") + } + + def statCityProcent(citys){ + def rets = [] + rets << ["城市","数据量","地址","地址别名","别名","楼栋总数","户数","产权年限","产权性质","建筑结构","竣工时间","小区当前均价"] + for(def c : citys){ + def dbo = CommonDatautils.findCityInfo(c) + if(dbo){ + def detail = dbo.district_detail + def ret = [] + ret << c + if(detail){ + ret << dbo.existing_district_count//数据量 + ret << detail.address*100// 地址 + ret << detail.addrAlias*100// 地址别名 + ret << detail.alias*100// 别名 + ret << detail.loudongCount*100// 楼栋总数 + ret << detail.householdCount*100// 户数 + ret << detail.propertyLife*100// 产权年限 + ret << detail.houseProperty*100// 产权性质 + ret << detail.buildStructure*100// 建筑结构(按材料划分) + ret << detail.finishBuildingDate*100// 竣工时间 + ret << dbo.price_coverage*100// 小区当前均价 + } + rets << ret + } + } + + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省城市字段覆盖率统计") + } + + def statCityHouseloudongProcent(citys){ + def rets = [] + + rets << ["城市","楼栋","单元","所在层数","房号","面积","套型","楼层"] + for(def c : citys){ + def dbo = CommonDatautils.findCityInfo(c) + if(dbo){ + def detail = dbo.house_loudong_detail + def ret = [] + ret << c + if(detail){ + ret << detail.house_num_rate*100//楼栋 + ret << detail.unit_num_rate*100// 单元 + ret << detail.room_num_rate*100// 所在层数 + ret << detail.room_num_rate*100// 房号 + ret << detail.area_rate*100// 面积 + ret << detail.room_info_rate*100// 套型 + } + def loudong_detail = dbo.loudong_detail + if(loudong_detail){ + ret << loudong_detail.loudong_height_rate*100// 楼层 + } + rets << ret + } + } + + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省房产信息字段覆盖率统计") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy new file mode 100644 index 000000000..a2ec7c80b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil + +class DevProcess { + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + static main(args) { + def task = new DevProcess(); + task.run() + } + def run(){ + def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls") + def index = 0 + + def mcaDistMap = CommonDatautils.queryDistNameCity() + for(def r : rets){ + index ++ + if(index >2){ + def udata = new BasicDBObject() + JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(null, r['E']); + if (jsonObject.get("district") && jsonObject.get("district") == "null") { + jsonObject = FangjiaClientUtils.formatFivelevel(null, r['C']+r['E']); + } + if (jsonObject.get("district") && jsonObject.get("district") == "null") { + jsonObject = FangjiaClientUtils.formatFivelevel(null, r['B']+r['C']+r['E']); + } + + udata.address = r['E'] + udata.create_time = new Date() + udata.industry = r['F'] + udata.level = "国家级" + udata.name = r['D'] + udata.province = r['B'] + udata.type = "农村创业园区" + + if(jsonObject) { + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + udata.city = jsonObject.get("city") + } + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + udata.district = jsonObject.get("district") + } + if(!udata.district && udata.city){ + def city = RegionData.formatCity(udata.city) + def region = RegionData.formatRegion(udata.city, r['C']) + if(region){ + def regionObj = mcaDistMap.get(city+"#"+region) + if(regionObj){ + udata.district = regionObj.district + } + } + } + if(jsonObject.get("township") && jsonObject.get("township")!="null"){ + udata.township = jsonObject.get("township") + } + + if(!udata.district){ + println index+" "+ udata + } + + } + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + + + } + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy new file mode 100644 index 000000000..2a638980a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -0,0 +1,467 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.PgsqlCommUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class GXDdistrictMatch { + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def regionCodeDataMap = PgsqlCommUtils.findDistrictMap(); + + def fullCityMap = CommonDatautils.queryFullNameCity(); + + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + + static main(args) { + def task = new GXDdistrictMatch(); +// task.run() +// task.addOldMatch() +// task.runDb() + +// task.updateDistStatus()//恢复name_cat不正常数据 +// task.saveToDB() //保存新增小区 + task.restoreDist()//恢复小区status=0 + } + + def addOldMatch(){ + //老匹配 + def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + + def runDb(){ + def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + ExcelUtils.modify(fileName, { d -> + if(!d['U']){ + def fullCityObj = fullCityMap.get(d['AD']) + if(fullCityObj){ + def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E']) + def obj = findPoiFromDB(fullCityObj.city, fullRegion, d['C']) + if(obj){ + d['U'] = "地图库" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.type + d['Z'] = obj.address + d['AA'] = obj.lng + d['AB'] = obj.lat + }else{ + obj = findDatas(d['AD'], d['AE'], d['C'], d['C'].replaceAll('[·\\(\\)()]|·', '')) + if(obj){ + d['U'] = "楼盘抓取库" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.property_type + d['Z'] = obj.address + def loc = obj.location + if(loc){ + d['AA'] = loc.x + d['AB'] = loc.y + } + } + } + } + } + }) + } + + def findPoiFromDB(city, region, name){ + def rets = EsSearchUtils.findPois(city, region, name) + def obj = null + if(rets && rets.size()==1){ + obj = rets[0] + } + return obj + } + + + def run(){ + def fileName = "D:\\数据\\国信达\\接口匹配.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['V'] && d['A']){ + def city = getCityFromCityCode(d['A']) + def region = regionCodeDataMap.get(d['A'] + "#" + d['E']) + if (city && region) { + region = RegionData.formatRegion(city, region) + d['AD'] = city + d['AE'] = region + if (region) { + d = match(city, region, d) + } else { + println d['A'] + "#" + d['E'] + " 区域无法格式化" + } + + } else { + println d['A'] + "#" + d['E'] + " 无法格式化" + } + } + }) + } + + def updateDistStatus(){ + def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx" + def index = 0 + ExcelUtils.modify(fileName, { d -> + if(d['U'] && ["名称匹配","名称匹配(Update)","名称在别名中","名称在别名中(Update)","别名匹配"].contains(d['U'])){ + index++ + + def status = d['Z'] + def name_cat = d['AA'] + + println d["V"]+" "+d["X"]+name_cat + + if(status!=null && status.toString().toDouble().toInteger()==0){ + if(name_cat==null){//&& name_cat.toString().toDouble().toInteger()!=0 + + def dist = CommonDatautils.findDistrict(d["V"],d["W"],d["X"], "district") + if(dist){ + def delUdata = new BasicDBObject() + delUdata.put("name_cat", 0) + delUdata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata)) + saveOptLog(dist.city, dist.region, dist.name, "update") + } + } + } + } + }) + } + + def restoreDist(){ + def fileName = "D:\\数据\\国信达\\11城小区需恢复数据.xlsx"//国信达11城(全).xlsx + def index = 0 + ExcelUtils.modify(fileName, { d -> + if(d['U']){ + index++ + def status = d['Z'] + println d["V"]+" "+d["X"] + if(status && status.toString().toDouble().toInteger()==1){ + def dists = CommonDatautils.findAllStatusDistrict(d["V"],d["W"],d["X"], "district") + if(dists && dists.size()==1){ + def dist = dists[0] + def delUdata = new BasicDBObject() + delUdata.put("name_cat", 0) + delUdata.put("status", 0) + delUdata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata)) + saveOptLog(dist.city, dist.region, dist.name, "update-status") + } + } + } + }) + } + + def saveToDB(){ + def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + ExcelUtils.modify(fileName, { d -> + if(d['U'] && d['AG']!='过滤'){ + def udata = new BasicDBObject() + udata.put("city", d['AD']) + udata.put("region", d['AE']) + def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def alias = [] + if(name!=d['C']){ + alias << d['C'] + } + if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ + alias << d['AI'] + } + if(alias){ + udata.put("alias", alias) + } + + udata.put("address", d['AF']) + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(d['AH']){ + type = d['AH'] + }else{ + type = d['Y'] + } + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(d['AA']){ + Long lng = d['AA']*10000000000 + udata.put("lng", lng) + } + if(d['AB']){ + Long lat = d['AB'] * 10000000000 + udata.put("lat", lat) + } + + def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) + def opt = "add" + if(s && s=='成功'){ + saveOptLog(udata.city, udata.region, udata.name, opt) + } + } + }) + + } + + + def match(city, region, d){ + d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "") + if(!d['U']){ + d = matching(city, region, d, d['C'].replace("小区","").replace("号楼","号院").replace("号","号院"), d['D'], "(Update)") + if(!d['U']){ + d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + + if(!d['U']){ + if(d['F'] && d['G']){ + d = matchAddr(city, region, d, "") + } + } + + if(!d['U']){ + d = matchApi(city, region, d, "property", false) + if(!d['U']){ + d = matchApi(city, region, d, null, true) + } + } + + if(!d['U']){ + d = matching2(city, region, d, d['C'], d['D'], "") + } + + if(!d['U']){ + if(d['C'].contains("小区")){ + d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") + }else{ + d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + } + } + return d + } + + def matchApi(city, region, d, cat, useMap){ + def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) + if(!jsonObject){ + jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap) + if(!jsonObject && d['D']){ + jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap) + } + } + if(jsonObject != null && jsonObject.credit>0) { + d['U'] = "接口匹配" + d['V'] = jsonObject.get("city") + d['W'] = jsonObject.get("district") + d['X'] = jsonObject.get("name") + def type = jsonObject.get("category")//propertyType + if(type && type!='null'){ + d['Y'] = type + } + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + } + d['AC'] = jsonObject.get("credit") + if(jsonObject.get("address") && jsonObject.get("address")!='null'){ + d['AH'] = jsonObject.get("address") + } + + } + return d + } + + + def matching(city, region, d, name, alias, str){ + //正常状态下有没有 + def rets = CommonDatautils.findDistrictList(city, region, name, "district") + if(rets){ + if(rets.size()>=1){ + d = setRowValue(rets[0], d, "名称匹配"+str) + } + }else{ + rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()>=1){ + d = setRowValue(rets[0], d, "名称在别名中"+str) + }else{ + rets = CommonDatautils.findDistrictByAddress(city, region, name, "district") + if(rets && rets.size()>=1){ + d = setRowValue(rets[0], d, "名称在地址中"+str) + }else{ + if (d['D']) { + rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district") + if (rets) { + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配"+str) + } + } + } + } + } + } + + if(rets){ + d['AI'] = rets.size() + } + return d; + } + + def matching2(city, region, d, name, alias, str){ + def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的 + if(rets){ + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "名称匹配(非正常)"+str) + } + }else{ + + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district") + if (rets && rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + }else{ + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district") + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + } + } + } + + if(rets){ + d['AI'] = rets.size() + } + return d; + } + + def matchAddr(city, region, d, str){ + def rets = CommonDatautils.findDistrictByAddress(city, region, d['F']+d['G'],"district") + if(rets){ + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "地址匹配") + } + } + if(rets){ + d['AI'] = rets.size() + } + return d + } + + def setRowValue(dbo, d, remark){ + if (dbo) { + d['V'] = dbo.city + d['W'] = dbo.region + d['X'] = dbo.name + d['Y'] = dbo.estate_type2 + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + d['U'] = remark + d['AH'] = dbo.address + } + return d; + } + + def getCityFromCityCode(cityCode){ + def city = null + for(def m : cityCodeDataMap){ + if(m.value == cityCode){ + city = m.key + break; + } + } + return city; + } + + def filterSpacialChar(def s){ + def pattern = ~/[()!;\·]\[【】()]/ + def matcher = pattern.matcher(s) + + def count = matcher.getCount() + if(count && count>0){ + return matcher.replaceAll('_') + } + return null; + } + + + def findDatas(def city, def region, def name1, def name2){ + def sql = "select * from community where city='${city}' and region='${region}' and (name='${name1}' or name='${name2}')" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + println d.property_type + }) + + if(rets){ + def obj = null + for(def r : rets){ + if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){ + obj = r + break + } + } + if(obj){ + return obj; + }else{ + return rets[0]; + } + + } + return null; + } + + def saveOptLog(city, region, name, type){ + def obj = new BasicDBObject(); + obj.put("city",city); + obj.put("region",region); + obj.put("district_name",name); + obj.put("category","district"); + obj.put("c_date", new Date()) + obj.put("m_date", new Date()) + obj.put("type", type) + obj.put("user_name", "高娟") + obj.put("origin", "script") + OPERATION_LOG_COLL.save(obj) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipPrcoess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipPrcoess.groovy new file mode 100644 index 000000000..f87cebb9c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipPrcoess.groovy @@ -0,0 +1,51 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.regex.Pattern + +class TownshipPrcoess { + + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + static main(args) { + def task = new TownshipPrcoess(); + task.run() + } + + def run() { + def fileName = "D:\\数据\\数据建设\\乡镇别名问题.xlsx" + ExcelUtils.modify(fileName, { d -> + if(!d['G']){ +// def name1 = d['E'].replace("街道","").replace("镇","").replace("乡","") +// def name2 = d['F'].replace("街道","").replace("镇","").replace("乡","") +// if(name1 == name2){ +// d['G'] = "不处理" +// } + + def lists1 = findCommunityTown(community_CONN, d['C'], d['D'], d['E']) + def lists2 = findCommunityTown(community_CONN, d['C'], d['D'], d['F']) + if(lists1 && lists2){ + d['G'] = "需删除" + } + } + }) + } + + + def findCommunityTown(community_CONN, city, region, town){ + def q = new BasicDBObject() + q.put("city_name", city) + q.put("area_name", region) + q.put("street_name", town) + def rets = community_CONN.find(q).toList() + return rets; + + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy new file mode 100644 index 000000000..7c9472e42 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -0,0 +1,13 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class TradeDatasDistBatch { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + static main(args) { +// def + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 3abfbaa22..260ef5cb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -228,9 +228,9 @@ class FangjiaClientUtils { def url = null; try { address = address.replace("{","").replace("\\","") - url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" -// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" - + url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") } @@ -314,13 +314,13 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") +// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) - println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy index 6f2307343..674198bd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.es import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ElasticUtils import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -16,7 +17,13 @@ class EsSelectTest { static main(args) { def task = new EsSelectTest() - task.run() +// task.run() + task.test() + } + + + def test(){ + def ret = CommonDatautils.findLogs("1007741", "/property/evaluate", "2024-03-01", "2024-03-28") } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy index 7d683c976..1196b6bd2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy @@ -18,7 +18,44 @@ DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) -def filePath = "D:\\数据\\光大\\全量.xlsx" +def filePath = "D:\\数据\\南京\\南京银行测试-新跑.xlsx" +ExcelUtils.multiThreadingModify(filePath, { row -> + def communityObj = null + if(row['E']){ + communityObj = findCommunityTown(community_CONN, row['C'], row['D'], row['E']) + } + if(communityObj){ + row['M'] = communityObj.province_code + row['N'] = communityObj.city_code + row['O'] = communityObj.area_code + row['P'] = communityObj.street_code + } + if(row['D']){ + if(row['D'].toString().endsWith("区")){ + row['F'] = "非农" + }else{ + if(communityObj){ + if(communityObj.has_gov && communityObj.has_gov=="是"){ + row['I'] = "是" + row['F'] = "非农" + }else{ + row['I'] = "否" + if(row['E']){ + def townObj = isTown(row['C'], row['D'], row['E']) + if(townObj){ + row['J'] = "是" + row['F'] = "非农" + }else{ + row['F'] = "农" + } + } + } + } + } + } + +}); +//def filePath = "D:\\数据\\光大\\全量.xlsx" //ExcelUtils.modify(filePath, { row -> // if(!row['O']){ // def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A']) @@ -62,19 +99,19 @@ def filePath = "D:\\ // } // } - def retMap = SignatureUtils.getAliAddressStrc(row['A']) - println row['A'] +" " + retMap.toString() - if(retMap){ - row['Y'] = retMap.prov - row['Z'] = retMap.city - row['AA'] = retMap.district - row['AB'] = retMap.town - row['AC'] = retMap.devzone - row['AD'] = retMap.road - row['AE'] = retMap.roadno - row['AF'] = retMap.poi - row['AG'] = retMap.subpoi - } +// def retMap = SignatureUtils.getAliAddressStrc(row['A']) +// println row['A'] +" " + retMap.toString() +// if(retMap){ +// row['Y'] = retMap.prov +// row['Z'] = retMap.city +// row['AA'] = retMap.district +// row['AB'] = retMap.town +// row['AC'] = retMap.devzone +// row['AD'] = retMap.road +// row['AE'] = retMap.roadno +// row['AF'] = retMap.poi +// row['AG'] = retMap.subpoi +// } // }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy new file mode 100644 index 000000000..dade0fa1f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -0,0 +1,191 @@ +package com.fangjia.script.gj.village + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.HttpClient +import net.sf.json.JSONObject +import okhttp3.*; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.concurrent.TimeUnit; + + +import okhttp3.Request.Builder; + +class QiliangTest { + +// static main(args) { +// def task = new QiliangTest() +//// task.run() +// task.getHttpResult("扬州扬子江北路399号11041") +// } + + def run(){//地址测试-2024.3.4-结果.xlsx + ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> + if(!row['N']){ + def address = null + if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ + address = row['B']+row['C'] + }else{ + address = row['C'] + } + if(address){ + def result = getHttpResult(address) + if(result && result.code==20000){ + def data = result.data + if(data){ + if(data.province && data.province!='null'){ + row['N'] = data.province + } + if(data.city && data.city!='null'){ + row['O'] = data.city + } + if(data.district && data.district!='null'){ + row['P'] = data.district + } + if(data.town && data.town!='null'){ + row['Q'] = data.town + } + if(data.village && data.village!='null'){ + row['R'] = data.village + } + + if(data.areaCode && data.areaCode!='null'){ + row['S'] = data.areaCode + } + + if(data.farmer == "0"){ + row['T'] = "非农" + }else{ + row['T'] = "农" + } + if(data.isGovt == "0"){ + row['U'] = "否" + }else{ + row['U'] = "是" + } + + def list = data.indusParkList + if(list && list!="null"){ + def s = "" + for(def r : list){ + s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" + } + row['V'] = s + } + if(data.govtAddr && data.govtAddr!='null'){ + row['W'] = data.govtAddr?data.govtAddr:'' + } + + } + } + } + } + }) + } + + def getHttpResult(address){ + def data = new JSONObject() + data.put("address", address) + data.put("type","2") + + Map headers = new HashMap(); + headers.put("Accept", "application/json, text/plain, */*"); + headers.put("Accept-Encoding", "gzip, deflate"); + headers.put("Accept-Language","zh-CN,zh;q=0.9") + headers.put("Content-Type","application/json;charset=UTF-8") + headers.put("Host","146.56.217.44:9010") + headers.put("Origin","http://146.56.217.44:8088") + headers.put("Proxy-Connection","keep-alive") + headers.put("Referer","http://146.56.217.44:8088/") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + + HttpRequest request1 = new HttpRequest() + try{ + String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") + def response = JSONObject.fromObject(res) + println response + return response + }catch(Exception e){ + e.printStackTrace() + } + return null; + } + + + final static String proxyIp = "tunnel2.qg.net"; + final static Integer proxyPort = 14186; + final static String authKey = "113BBC06"; + final static String password = "3B2FBD05B0AF"; + public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8"); + +// public String request(params) throws IOException { +// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort)); +// OkHttpClient client = new OkHttpClient.Builder() +// .proxy(proxy) +// .proxyAuthenticator((route, response){ +// String credential = Credentials.basic(authKey, password); +// return response.request().newBuilder().header("Proxy-Authorization", credential).build(); +// }). +// build(); +//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build(); +// +// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params)); +// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build(); +// +// Response response = client.newCall(request).execute(); +// return response.body().string(); +// +//// return client.newCall(request).execute(); +// } + public static void main(String[] args) { + def proxyHost = "tunnel2.qg.net" + def proxyPort = 14186 + def proxyUsername = '113BBC06' + def proxyPassword = '3B2FBD05B0AF' + + def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign' + def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2'] + + +// 创建代理服务器的连接 + def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}") + +// 打开代理连接 + def proxyConnection = proxy.openConnection() + +// 创建目标URL的连接 + def url = new URL(targetUrl) + def connection = url.openConnection(proxy) + +// 设置请求方法 + connection.setRequestMethod("GET") + +// 添加请求头(可选) + connection.setRequestProperty("Content-Type", "application/json") + +// 构建查询参数 + def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&') + +// 将查询参数附加到目标URL + def finalUrl = "${targetUrl}?${queryString}" + +// 发送请求 + connection.connect() + +// 获取响应 + def responseCode = connection.getResponseCode() + def responseMessage = connection.getResponseMessage() + +// 读取响应内容 + def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) + def responseContent = reader.readLine() + reader.close() + + println "Response Code: ${responseCode}" + println "Response Message: ${responseMessage}" + println "Response Content: ${responseContent}" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy index 6dd3714bb..b2dbb9204 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy @@ -200,4 +200,9 @@ public class SignatureUtils { } return null; } + + public static void main(String[] args){ + def a = getAliAddressStrc("上海市长宁区新泾三村") + println a + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index d5d8303d5..cfda2228d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -66,8 +66,10 @@ class GaoDePoiDataProcess extends BaseBatch{ @Override void process(Object args) { // println ProvinceData.getProvince("成都") - synchMapPoi(); -// synchCompany() +// synchMapPoi(); + + + synchCompany() // poiCatProcess() // addressAndCatProcess() // synchNetwork() @@ -99,10 +101,10 @@ class GaoDePoiDataProcess extends BaseBatch{ def synchCompany(){ fullCityMap = CommonDatautils.queryFullNameCity() //"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市", - def citys = ["湖州市"] + def citys = ["杭州","宁波","温州","嘉兴","台州","绍兴","金华","舟山","衢州","丽水"] for(def sampleCity : citys){ - def cityObj = fullCityMap.get("湖州"); + def cityObj = fullCityMap.get(sampleCity); def city = null if(cityObj){ city = cityObj.city @@ -112,6 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{ println city+ " 加载公司数据:"+companyDatas.size() def rows = running(city, companyDatas, "company", townMap) save(city, rows) +// saveToMG(city, rows) println "完成" } } @@ -182,7 +185,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def createCompanyData(map, townMap){ def retMap = [:] - retMap._id = map._id + retMap.data_id = map._id.toString() retMap.fjw_province = map.province_name retMap.fjw_city = map.city_name diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 8aa6511ac..2896b5c0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong class SuggestionDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/seaweed/" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy index 3579922b1..c7a13eea0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy @@ -45,10 +45,10 @@ class PdsUseStatisticBatch extends BaseBatch{ final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ - "duweiliang@fangjia.com", - "yaojingmeng@fangjia.com", - "xuqi@fangjia.com", - "wangpeili@fangjia.com" +// "duweiliang@fangjia.com", +// "yaojingmeng@fangjia.com", +// "xuqi@fangjia.com", + "gaojuan@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy index 7ec752606..26b8c5aa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy @@ -163,9 +163,9 @@ class AnalyticsApiLogPerformanceBatch extends BaseBatch { } static void sendEmail(String sub, String msg) { - jms.sendMail("duxiaoyu@fangjia.com", sub, msg) +// jms.sendMail("duxiaoyu@fangjia.com", sub, msg) jms.sendMail("wangwenyan@fangjia.com", sub, msg) - jms.sendMail("helvyuan@fangjia.com", sub, msg) +// jms.sendMail("helvyuan@fangjia.com", sub, msg) jms.sendMail("gaojuan@fangjia.com", sub, msg) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy index 30da55b3e..94ea4a405 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy @@ -32,11 +32,11 @@ class CommonLogAnaBatch extends BaseBatch { def static bjEvaReport = []; public static final def EMAILS = [ - "jiangxiaochun@fangjia.com", - "limingmin@fangjia.com", - "xiaohai@fangjia.com", - "luxuping@fangjia.com", - "yukai@fangjia.com" + "jiangxiaochun@fangjia.com" +// "limingmin@fangjia.com", +// "xiaohai@fangjia.com", +// "luxuping@fangjia.com", +// "yukai@fangjia.com" ]; static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy index bd412eda3..41c9da3f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy @@ -26,12 +26,12 @@ class DaikuanLogAnaBatch extends BaseBatch { static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); public static final def EMAILS = [ - "jiangxiaochun@fangjia.com", - "limingmin@fangjia.com", - "zhongjingyi@fangjia.com", - "xiaohai@fangjia.com", - "luxuping@fangjia.com", - "yukai@fangjia.com" + "jiangxiaochun@fangjia.com" +// "limingmin@fangjia.com", +// "zhongjingyi@fangjia.com", +// "xiaohai@fangjia.com", +// "luxuping@fangjia.com", +// "yukai@fangjia.com" ]; static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy index b2cda8f17..b2d649195 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy @@ -148,14 +148,14 @@ class ReadBkoffSafeLogBatch extends BaseBatch{ } static void sendEmail(String sub, String msg) { - jms.sendMail("duxiaoyu@fangjia.com", sub, msg) - jms.sendMail("joe@fangjia.com", sub, msg) - jms.sendMail("xiaohai@fangjia.com", sub, msg) +// jms.sendMail("duxiaoyu@fangjia.com", sub, msg) +// jms.sendMail("joe@fangjia.com", sub, msg) +// jms.sendMail("xiaohai@fangjia.com", sub, msg) jms.sendMail("caoyuyan@fangjia.com", sub, msg) - jms.sendMail("wangchencen@fangjia.com", sub, msg) - jms.sendMail("yaojingmeng@fangjia.com", sub, msg) - jms.sendMail("chichilook@qq.com", sub, msg) - jms.sendMail("sunjian@fangjia.com", sub, msg) +// jms.sendMail("wangchencen@fangjia.com", sub, msg) +// jms.sendMail("yaojingmeng@fangjia.com", sub, msg) +// jms.sendMail("chichilook@qq.com", sub, msg) +// jms.sendMail("sunjian@fangjia.com", sub, msg) } static public String getCatById(String id) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy new file mode 100644 index 000000000..cb9ef1193 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy @@ -0,0 +1,50 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.village.ailiyun.SignatureUtils +import com.fangjia.script.util.ExcelUtils + +//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801" +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\1.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + + count = count + 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + sleep(80) + + def address = items['A'] + def map = SignatureUtils.getAliAddressStrc(address) + //prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012 + if ("prov" in map) { + def province = map.get("prov") + items['E'] = province + } + if ("city" in map) { + def city = map.get("city") + items['G'] = city + } + if ("district" in map) { + def district = map.get("district") + items['I'] = district + } + if ("town" in map) { + def town = map.get("town") + items['K'] = town + } +// if ("community" in map) { +// def community = map.get("community") +// items['S'] = community +// } + if ("devzone" in map) { + def devzone = map.get("devzone") + items['Q'] = devzone + + } + +}) + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy new file mode 100644 index 000000000..7a2098677 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.majintao.JSBankTest; + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils; +import net.sf.json.JSONObject; +import java.util.Base64; +import net.sf.json.JSONArray; + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count += 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + def zx = ["北京", "天津", "上海", "重庆"] +// String province = "天津"; + String province = items['E']; + if (zx.contains(province)) { + province = province + "市"; + } +// String name = "天津经济技术开发区"; + String name = items['Q']; + String elasticUsername = "elastic"; + String elasticPassword = "elasticsearch"; + + String seawaterUrl = "http://192.168.0.89:9200/development_zone/_search"; + String basicAuth = "Basic " + Base64.getEncoder().encodeToString((elasticUsername + ":" + elasticPassword).getBytes("UTF-8")); + + String body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "name": { + "query": "${name}", + "operator": "and" + } + } + }, + { + "term": {"province": {"value": "${province}"}} + } + ] + } + } + } + """; + + HttpRequest request = new HttpRequest(); + JSONObject jsonBody = JSONObject.fromObject(body); + JSONObject header = new JSONObject(); + header.put("Authorization", basicAuth); + String res = request.postWithBody(seawaterUrl, jsonBody, header, "UTF-8"); + + JSONObject response = JSONObject.fromObject(res); + JSONObject obj = null; + if (response != null && response.containsKey("hits")) { + JSONObject hits = response.getJSONObject("hits"); + if (hits != null && hits.size() > 0) { + JSONArray hitsArray = hits.getJSONArray("hits"); + // 省略了之前的部分... + + if (hitsArray != null && hitsArray.size() > 0) { + for (int i = 0; i < hitsArray.size(); i++) { + JSONObject hit = hitsArray.getJSONObject(i); + JSONObject source = hit.getJSONObject("_source"); + String devName = source.optString("name"); // Assuming "name" is a String + String level = source.optString("level"); // Assuming "level" is a String + if (devName.equals(name)) { + items['O'] = "是"; + items['P'] = level; + println("省份:" + province + " 名称:" + name + " 开发区名:" + devName + " 开发区级别:" + level); + break + } + } + } + + } + } +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy new file mode 100644 index 000000000..438897f64 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.majintao.JSBankTest + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import org.apache.tools.ant.types.resources.selectors.None + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count += 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + + + String address = items['B'] + + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + + def map = [:] + map.put("address", address) + map.put("showFullValue", "2") + map.put("showTestValue", "2") + map.put("type", "2") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if (result != null) { + if ("addrVerify") { + items['C'] = result.get("addrVerify") + } + + if ("province" in result) { + items['D'] = result.get("province") + } + if ("city" in result) { + items['F'] = result.get("city") + } + if ("district" in result) { + + items['H'] = result.get("district") + } + if ("town" in result) { + items['J'] = result.get("town") + } + if ("committee" in result) { + + items['L'] = result.get("committee") + } + if ("typeCode" in result) { + items['M'] = result.get("typeCode") + } + if ("countyGov") { + items['N'] = result.get("countyGov") + } + def devDistrict = "" + if ("devDistrict" in result) { + devDistrict = result.get("devDistrict") + } + def dev = "" + if ("dev" in result) { + dev = result.get("dev") + } + // 开发区级别暂时没有 + + // 暂时使用 + def defGrade = "" + if (!dev.equals("") && devDistrict.equals("否")) { + defGrade = "非国际级或省级" + devDistrict = "是" + } + if (!devDistrict.equals("")) { + items['O'] = devDistrict + } + items["P"] = defGrade + } + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_CommunicateLog_info b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_CommunicateLog_info similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_CommunicateLog_info rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_CommunicateLog_info diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_EMAYSDK_info b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_EMAYSDK_info similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_EMAYSDK_info rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_EMAYSDK_info diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy index 1ac784f05..bfc2a005d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy @@ -5,7 +5,11 @@ import com.fangjia.script.gj.village.ailiyun.SignatureUtils import com.fangjia.script.util.ExcelUtils //def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801" +<<<<<<< HEAD +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx"; +======= String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx"; +>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 ExcelUtils.removeEmptyRow(excelFilePath) def count = 0 ExcelUtils.singleThreadingModify(excelFilePath, { List items -> @@ -14,13 +18,43 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items -> if (count % 10 == 0) { println("已处理:" + count) } +<<<<<<< HEAD + sleep(80) +======= sleep(50) +>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 def address = items['A'] def map = SignatureUtils.getAliAddressStrc(address) //prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012 if ("prov" in map) { def province = map.get("prov") +<<<<<<< HEAD + items['C'] = province + } + if ("city" in map) { + def city = map.get("city") + items['G'] = city + } + if ("district" in map) { + def district = map.get("district") + items['K'] = district + } + if ("town" in map) { + def town = map.get("town") + items['O'] = town + } + if ("community" in map) { + def community = map.get("community") + items['S'] = community + + } + if ("devzone" in map) { + def devzone = map.get("devzone") + items['AE'] = devzone + + } +======= items['D'] = province } if ("city" in map) { @@ -66,6 +100,7 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items -> // items['K'] = roadno // } +>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy index c1575a52f..91967ecf4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy @@ -4,10 +4,17 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils +<<<<<<< HEAD +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> +======= String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx"; ExcelUtils.removeEmptyRow(excelFilePath) def count = 0 ExcelUtils.multiThreadingModify(excelFilePath, { List items -> +>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 count += 1 if (count % 10 == 0) { println("已处理:" + count) @@ -15,6 +22,112 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items -> String address = items['A'] +<<<<<<< HEAD +// if (province == null) { + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + + def map = [:] + map.put("address", address) + map.put("showFullValue", "2") + map.put("showTestValue", "2") + map.put("type", "2") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if (result != null) { + if ("province" in result) { + items['B'] = result.get("province") + } + if ("provinceCode" in result) { + items['E'] = result.get("provinceCode") + } + if ("city" in result) { + items['F'] = result.get("city") + } + if ("cityCode" in result) { + items['I'] = result.get("cityCode") + } + if ("district" in result) { + + items['J'] = result.get("district") + } + if ("districtCode" in result) { + + items['M'] = result.get("districtCode") + } + if ("town" in result) { + + items['N'] = result.get("town") + } + if ("townCode" in result) { + + items['Q'] = result.get("townCode") + } + + if ("committee" in result) { + + items['R'] = result.get("committee") + } + if ("committeeCode" in result) { + + items['U'] = result.get("committeeCode") + } + if ("typeCode" in result) { + + items['V'] = result.get("typeCode") + } + if ("typeName") { + items['W'] = result.get("typeName") + } + if ("verifyFlag") { + items['X'] = result.get("verifyFlag") + } + if ("addrVerify") { + items['Y'] = result.get("addrVerify") + } + if ("countyLevelCity") { + items['Z'] = result.get("countyLevelCity") + } + if ("countyGov") { + items['AA'] = result.get("countyGov") + } + if ("communitySource") { + items['AB'] = result.get("communitySource") + } + + if ("administrativeTowns" in result) { + items['AD'] = result.get("administrativeTowns") + } + if ("devDistrict" in result) { + items['AF'] = result.get("devDistrict") + } + if ("devDistrictName" in result) { + items['AG'] = result.get("devDistrictName") + } + if ("splitCommittee" in result) { + items['AH'] = result.get("splitCommittee") + } + if ("splitStreet" in result) { + items['AI'] = result.get("splitStreet") + } + if ("poiName" in result) { + items['AJ'] = result.get("poiName") + } + if ("splitStreetNum" in result) { + items['AK'] = result.get("splitStreetNum") + } + if ("dev" in result) { + items['AL'] = result.get("dev") + } + } + +// } + +======= String province = items['B'] if (province == null) { String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; @@ -120,5 +233,6 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items -> } +>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy index f59dcafe2..40ff50c4e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy @@ -24,8 +24,13 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items -> def map = [:] map.put("address", address) +<<<<<<< HEAD + map.put("complete", "2") + map.put("addressAssessment", "2") +======= map.put("complete", "true") map.put("addressAssessment", "true") +>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 map.put("type", "2") def content = httpRequest.getJsonContent(url, map, "UTF-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy new file mode 100644 index 000000000..9f339f5f0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy @@ -0,0 +1,15 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest + +String url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify"; + +HttpRequest httpRequest = new HttpRequest(); +map = new HashMap() + + +def map = [:] +map.put("address", address) +map.put("showFullValue", "2") +map.put("showTestValue", "2") +map.put("type", "2") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 49136a392..42972bc05 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -23,8 +23,8 @@ class DataStat { // for(def city :citys){ // println city // } -// task.run(); - task.stat() + task.run(); +// task.stat() // def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"] // def rets = [] @@ -76,7 +76,7 @@ class DataStat { } def stat(){ - def dates = [202301,202302,202303,202304,202305,202306,202307,202308,202309,202310,202311,202401] + def dates = [202303,202304,202305,202306,202307,202308,202309,202310,202311,202401,202402] def sources = ["时间","我爱我家","兔博士","吉屋","贝壳找房","乐有家","365淘房","诸葛找房","链家","Q房","麦田","中原地产","安居客","搜房","C21","城市房产","幸福里"] // crawlCount(dates,sources); matchCount(dates,sources); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 7c9ed802c..33e407462 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,8 +26,8 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" -// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] - def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"] +// def esSources = ["搜房":"搜房"] + def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"] static main(args) { runBatch(new SynchDistrictPrice(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index e396b2e96..a53d2d7c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"] for(def source :sources){ - def count1 = findTotalCount(source, 202307); - def count2 = findTotalCount(source, 202401); + def count1 = findTotalCount(source, 202401); + def count2 = findTotalCount(source, 202402); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy index cd31567e6..448d8aaea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy @@ -36,10 +36,10 @@ class DBHelper { static final DB offline61PoseidonDb = new Mongo("192.168.0.61", 27037).getDB("poseidon"); static final online42GoojiaFangjiaDbConn = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?useUnicode=true&characterEncoding=utf-8", "sa", "G00jia", "com.mysql.jdbc.Driver"); static final offline89GoojiaFangjiaStatDbConn = Sql.newInstance("jdbc:mysql://192.168.0.89:3306/goojia_fangjia_stat?useUnicode=true&characterEncoding=utf-8", "root", "123456", "com.mysql.jdbc.Driver"); - static final emailsGroup0 = ['gaojuan@fangjia.com','yukai@fangjia.com','joe@fangjia.com','duxiaoyu@fangjia.com','tangye@fangjia.com','ruanyadong@fangjia.com','xiamin@fangjia.com','zhutianli@fangjia.com']; + static final emailsGroup0 = ['gaojuan@fangjia.com']; static final emailsFangjiaWang = ['all.list@fangjia.com']; static final mobilesYanzhou = ['13621626685']; - static final emailsYanzhou = ['duxiaoyu@fangjia.com']; + static final emailsYanzhou = ['wangwenyan@fangjia.com']; static List getCityNames(){ DBObject dbObject = new BasicDBObject("status",0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 8af7a4d6c..50882747b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 3 + def nThread = Runtime.getRuntime().availableProcessors() * 4 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) @@ -454,8 +454,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = 10 - println nThread + ExecutorService executorService = Executors.newFixedThreadPool(3) final AtomicInteger counter = new AtomicInteger(0) final double total = body.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy index 0f3c5e23a..9236574d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy @@ -23,7 +23,7 @@ public class ComparisonDataStatistics extends com.fangjia.script.BaseBatch{ static final DATE_NUMBER = new Date().format("yyyyMMdd").toInteger() static final DBCollection MESSENGER_COLL = MongoUtils.getCollectionByName("fangjia", "messenger", MongoUtils.FANGJIA_ONLINE_MONGO ,27017) static final def categories = ["district":"住宅", "office":"写字楼" , "shop":"商铺"] - def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","caoyuyan@fangjia.com"] + def emails = ["wangwenyan@fangjia.com","caoyuyan@fangjia.com"] static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); def static main(args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy index e7a3c1c7e..7241889ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy @@ -14,7 +14,7 @@ class EvaluationSourceAnalyze extends BaseBatch { static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); public static final DBCollection STAT_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") - def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","xiaohai@fangjia.com"] + def emails = ["wangwenyan@fangjia.com","xiaohai@fangjia.com"] static main(args) { new EvaluationSourceAnalyze().process(null); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy index d2ba9d116..f4430e6a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy @@ -1,7 +1,16 @@ +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.Mongo3Utils +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject import org.bson.Document +import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection @@ -11,67 +20,43 @@ import java.util.function.Consumer import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.eq +//REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); +//TOWNSHIP_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); +//TOWNSHIP_CONN.find().each {BasicDBObject township-> +// township.area_station = township.central +// township.removeField("central") +// TOWNSHIP_CONN.save(township) +// +//} +//REGION_CONN.find(new BasicDBObject("category", "region")).each {BasicDBObject regionBlock-> +// def lng = regionBlock.lng/10000000000l +// def lat = regionBlock.lat/10000000000l +// HttpRequest request = new HttpRequest() +// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8") +// if(resp.result){ +// TOWNSHIP_CONN.update(new BasicDBObject("_id", new ObjectId(resp.result.id)), new BasicDBObject('$set', new BasicDBObject('central', 1))) +// println(resp.result) +// } +//} -def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"] -def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"] -MongoCollection DISTRICT_STAT = Mongo3Utils.getDbCollection("fangjia", "district_stat_rent") -MongoCollection DEAL_PRICE = Mongo3Utils.getDbCollection("fangjia", "deal_price") -def data = [["城市", "区域", "小区名", "版块", "装修", "朝向", "室", "厅", "卫", '标题', '总价', '面积', '单价', '发布时间', '来源']] -cityList.each { def city -> - def statMap = [:] - DISTRICT_STAT.find(and(eq("city", city), eq("end_month_date", Date.parse("yyyyMMdd", "20190228")))).forEach(new Consumer() { - @Override - void accept(Document t) { - def key = "${t.city}_${t.region}_${t.district_name}_${t.house_type}".toString() - statMap[key] = t.avg_price - } - }) - roots.each { def root -> - EsDataSource dataSource = new EsDataSource(); - String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - dataSource.setUrl(address) - Properties connectionProperties = new Properties() - connectionProperties.put("user", "developer") - connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") - dataSource.setProperties(connectionProperties) - Connection connection = dataSource.getConnection() - Statement statement = connection.createStatement() - ResultSet rs = statement.executeQuery("""select * from "seawater-2019" where city='${city}' and cat = 'rent' order by p_date""") - int count = 0 - while (rs.next()) { - String region = rs.getString("region") - String district_name = rs.getString("district_name") - int room = rs.getInt("room") - def key = "${city}_${region}_${district_name}_${room}".toString() - if (!statMap.containsKey(key)) { - continue - } - int total_price = rs.getInt("total_price") - double area = rs.getDouble("area") - int avg_price = (int) (total_price / area) - if (total_price * 1.3 || total_price < statMap[key] * 0.7) { - continue - } - String block = rs.getString("block") - String name = rs.getString("name") - String fitment = rs.getString("fitment") - String direction = rs.getString("direction") - int hall = rs.getInt("hall") - int toilet = rs.getInt("toilet") - int p_date = rs.getInt("p_date") - if (!block || !name || !fitment || !direction || room == 0 || hall == 0 || toilet == 0) { - continue - } - println(rs.getString('url')) - def item = [city, region, district_name, block, fitment, direction, room, hall, toilet, name, total_price, area, avg_price, p_date, root] - println(item) - data << item - count++ - if(count > 10){ - break - } - } - } -} -ExcelUtils.write("D:\\建设银行\\租金样本.xlsx", data, "结果") \ No newline at end of file +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city, null, null, "district"),{BasicDBObject office-> +// if(!office.township){ +// def lng = office.lng/10000000000l +// def lat = office.lat/10000000000l +// HttpRequest request = new HttpRequest() +// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8") +// if(resp.result){ +// println(office._id.toString() + "\t" + office.township + "\t-->\t" + resp.result.streetName) +// DbUtils.seaweed.update(new BasicDBObject("_id", office._id), new BasicDBObject('$set', new BasicDBObject('township', resp.result.streetName))) +// } +// } +// }) +//} +ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{List item-> + def lng = item["AB"].toString().split(",")[0].toDouble() + def lat = item["AB"].toString().split(",")[1].toDouble() + item["AD"] = GeoUtils.calculateDistance(121.526672,31.142637,lng, lat) + println(item) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy index d0134fa35..d20623baa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy @@ -3,9 +3,9 @@ package com.fangjia.script.wenyan public enum Watcher { //在职同事 - DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), + DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'), - Tomzhao('15021021464','zhaoshumao@fangjia.com'), + Tomzhao('15021021464','wangwenyan@fangjia.com'), GaoJuan('15021384995','gaojuan@fangjia.com'); private final String mobile; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index ff32247d7..ce4d04b65 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -1,51 +1,15 @@ package com.fangjia.script.wenyan.test -import com.aliyuncs.DefaultAcsClient -import com.aliyuncs.IAcsClient -import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest -import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse -import com.aliyuncs.profile.DefaultProfile -import com.fangjia.core.util.HttpRequest -import com.fangjia.script.util.ExcelUtils -import net.sf.json.JSONObject - -def strt(String addr){ - -} - - -ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> - def city = item["A"] - def address1 = item["B"] - def address2 = item["C"] - HttpRequest http = new HttpRequest() - def params = new HashMap() - params["city"] = city - params["address_1"] = address1 - params["address_2"] = address2 - params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") - JSONObject result = json.result - item["D"] = result?.level - item["E"] = result?.formattedAddress_1 - item["F"] = result?.formattedAddress_2 - item["G"] = result?.isSameAddress - DefaultProfile profile = DefaultProfile.getProfile( - "cn-hangzhou", - "LTAIdaj4eSO79gzz", - "xw6EsvqydZ9HC6zAvIB3ruQqVHynxf"); - IAcsClient client = new DefaultAcsClient(profile); - GetAddressSimilarityRequest request = new GetAddressSimilarityRequest(); - request.setServiceCode("addrp") - request.setAppKey("Xcgtyhupey0h") - request.setText("${address1};${address2}") - GetAddressSimilarityResponse response = client.getAcsResponse(request) - String data = response.getData() - json = JSONObject.fromObject(data) - item["H"] = json["sim_level"]["level"] - item["I"] = json["sim_level"]["address_a"] - item["J"] = json["sim_level"]["address_b"] - item["K"] = json["sim_level"]["is_same"] - println(json) -}) +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +//、DSCollection seabird = new DataStore("192.168.0.38", 27007).getCollection("fangjia.seaweed") +//seabird.find(DbUtils.buildSeaweedQuery("常州", "溧阳", "吾悦广场").append("if_sell", 1)).each { +// println(it) +//} +long l = 0 +if(!l){ + println(l) +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy index 4d695b799..366adad0f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy @@ -27,8 +27,7 @@ class ApiDailyCountOfficialAccount extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') def static emails = [ - 'xuqi@fangjia.com', - 'zongzhimin@fangjia.com' + 'wangwenyan@fangjia.com' ] static void main(String[] args){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java index af7d6be61..1f7ab6869 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java @@ -165,8 +165,11 @@ public class SellTriggerServiceImpl implements TriggerService { newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); newObj.put("district_name", district.getName()); } + }else{ + if(object.containsField("city") && object.containsField("district_name") && object.containsField("region")){ + newObj.put("districtname_key", object.get("district_name") + "_" + object.get("region") + "_" + object.get("city")); + } } - // newObj.put("if_school", 0); // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { // List skList = districtSchoolMap.get(districtKey);