diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index dc3ee5d27..9be5b8c21 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -21,6 +21,7 @@ import org.bson.types.ObjectId /** * 通知计算二手房均价 + * 发送平米租金命令 * @author duxiaoyu * */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 8bc6aff7c..fca28ce3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -387,26 +387,43 @@ class DealPriceDataProcess { } def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys_all +// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" + def citys = ["469005"]//CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() - for(def city : citys) { - def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 - def total = 0 - println city +" "+rets.size() + for(def cityid : citys) { + def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 + println cityid +" "+rets.size() if(rets){ - set.add(city) + set.add(cityid) } for(def m : rets){ - def cityId = cityCodeDatas.get(city) +// def cityId = cityCodeDatas.get(city) // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // sqlInstance236.executeUpdate(sql) + def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + def total = 0 + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println cityid +" "+m.commid+" (按价格)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + pgsqlInstance.executeUpdate(sql) + }else{ - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - pgsqlInstance.executeUpdate(sql) - + querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println cityId +" "+m.commid+" (按小区)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.executeUpdate(sql) + } + } } } println set.toString() @@ -432,8 +449,8 @@ class DealPriceDataProcess { return rets } - def findPeGteThirdParty2(cityName, yearmonth){ - def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>20" + def findPeGteThirdParty2(cityid, yearmonth){ + def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20" def rets = [] pgsqlInstance.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index d2ad41623..00a1ddd5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -164,7 +164,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" @@ -251,7 +251,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDSTRU_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'") @@ -309,7 +309,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMHOUSETYPE_FJW 数据文件输出完成......" @@ -367,7 +367,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDYEAR_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'") @@ -430,7 +430,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMFEE_FJW 数据文件输出完成......" @@ -509,7 +509,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMHOUSEUSE_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'") @@ -595,7 +595,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_POSITION_FJW 数据文件输出完成......" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 1af08d6b3..e0e58814f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -40,6 +40,8 @@ import java.util.zip.GZIPInputStream class CommonDatautils { + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -3812,6 +3814,135 @@ class CommonDatautils { return rets; } + + + static def findHouses2(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range) + + if(region){ + queryBuilder.must(QueryBuilders.termQuery("region", region)) + } + + if(name || alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + def avgPrices = [] + def areas = [] + def totalPrices = [] + + if(houses){ + for (int k=0;k1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } + avgPrices << seawater.avg_price + areas << seawater.area + totalPrices << seawater.total_price + } + } + + def retMap = [:] + if(areas){ + retMap.avg_price = avgPrices + retMap.area = areas + retMap.total_price = totalPrices + } + return retMap; + } + + /** + * 是否是别墅 + * @return + */ + public static boolean isVilla(Double area, String name, Integer height, Integer room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + }else if(height != null){ + if (height <= 3) { + if (area > 200){ + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + static def parseFloor(def floorDesc){ if(floorDesc){ floorDesc = floorDesc.replaceAll("[\\s ]+", "") @@ -4081,7 +4212,7 @@ class CommonDatautils { def map = houses[k]; if(map.parameter_map){ def pMap = map.parameter_map; - if((pMap.keyword && pMap.keyword.contains("碧桂园")) || (pMap.name && pMap.name.contains("碧桂园"))){ + if((pMap.keyword && pMap.keyword.contains("紫金山蘭园")) || (pMap.name && pMap.name.contains("紫金山蘭园"))){ println map } } @@ -4319,7 +4450,20 @@ class CommonDatautils { if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ dataMap.put("maitian_esf_price", db.maitian_esf_price) } + if(!dataMap.containsKey("xingfuli_esf_price") && db.xingfuli_esf_price && db.xingfuli_esf_price>3000){ + dataMap.put("xingfuli_esf_price", db.xingfuli_esf_price) + } + if(!dataMap.containsKey("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } } return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index baccfa579..2b96edc91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.fuzhou import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -21,7 +22,38 @@ class FzDistrictMatch { // task.addPrice() // task.addThirdPrice() - task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") +// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") + task.longyanPrice() + } + + def longyanPrice(){ +// def dists = CommonDatautils.findAllDistricts("龙岩","district", ["city","region","name","estate_type2"]) +// def rets = [] +// rets << ["Id", "城市","区域", "名称", "物业类型", "上个月价格toB","上个月价格","最新价格toB","最新价格"] +// for(def d : dists){ +// def ret = [] +// ret << d._id.toString() +// ret << d.city +// ret << d.region +// ret << d.name +// ret << d.estate_type2 +// def obj2 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-03-10"), 1) +// if(obj2){ +// ret << obj2.avg_price_2b +// ret << obj2.avg_price +// } +// +// def obj1 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-04-07"), 1) +// if(obj1){ +// ret << obj1.avg_price_2b +// ret << obj1.avg_price +// } +// rets << ret +// } +// ExcelUtils.write("D:\\数据\\福州\\龙岩价格对比.xlsx", rets, "龙岩") + + + addThirdPrice("D:\\数据\\福州\\龙岩价格对比.xlsx") } def addThirdPrice2(path){ @@ -207,4 +239,57 @@ class FzDistrictMatch { } },0) } + + + def addThirdPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['C'] && d['C']=="新罗"){ + def cityMap = CommonDatautils.findThirdPartyPrice(d['B'], d['C'], d['D'], 202312, 202403) + def priceMap = cityMap.get(d['B']+"##"+d['C']+"##"+d['D']) + + if(priceMap && priceMap.zhugezhaofang_esf_price){ + d['J'] = priceMap.zhugezhaofang_esf_price + } + + if(priceMap && priceMap.anjuke_esf_price){ + d['K'] = priceMap.anjuke_esf_price + } + + if(priceMap && priceMap.soufang_esf_price) { + d['L'] = priceMap.soufang_esf_price + } + + if(priceMap && priceMap.lianjia_esf_price){ + d['M'] = priceMap.lianjia_esf_price + } + + if(priceMap && priceMap.cityhouse_esf_price){ + d['N'] = priceMap.cityhouse_esf_price + } + + if(priceMap && priceMap.zhongyuan_esf_price){ + d['O'] = priceMap.zhongyuan_esf_price + } + + if(priceMap && priceMap.xingfuli_esf_price){ + d['P'] = priceMap.xingfuli_esf_price + } + if(priceMap && priceMap.tuboshi_esf_price){ + d['Q'] = priceMap.tuboshi_esf_price + } + + if(priceMap && priceMap.get("5i5j_esf_price")){ + d['R'] = priceMap.get("5i5j_esf_price") + } + + def sellMap = CommonDatautils.findHouses2(d['B'], d['C'], "sell", 20231001, 20240407, d['D'], null, d['E']) + if(sellMap && sellMap.avg_price){ + if(sellMap.avg_price.size()>=3){ + d['S'] = Math.round(sellMap.avg_price.sum() / sellMap.avg_price.size()); + } + } + } + },0) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 9e365230a..ad2cf8430 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 28fb38a92..cb2fcce9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -20,16 +20,16 @@ class AddAvgpriceBatch { public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static { - citys = ["徐州"]//CommonDatautils.findAllCity() + citys = ["宜昌"]//CommonDatautils.findAllCity() println "加载城市完成:"+citys.size() } static main(args) { def task = new AddAvgpriceBatch(); -// task.run1(); + task.run1(); // task.test() // task.sendPriceCommand() - task.sendNewDistPrice() +// task.sendNewDistPrice() } def sendNewDistPrice(){ @@ -107,7 +107,7 @@ class AddAvgpriceBatch { println "------------------------------------------------------------" println "${city}小区量:"+districtKeysMap.size() if(districtKeysMap){ - def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26")) + def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2024-04-07")) println "【2023-03-26】共计均价小区:"+lastWeekendStats.size() //库中小区没有计算过均价的小区 def districtKeys = districtKeysMap.keySet(); @@ -118,7 +118,7 @@ class AddAvgpriceBatch { def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) //发重算均价命令 - addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd')) + addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2024-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2024-04-07','yyyy-MM-dd')) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index e6812a2e4..82893abe6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -23,18 +23,22 @@ class EsSearchUtils { ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString()) def rets = [] ElasticUtils.executeSqlQuery(sql,{ data-> - def m = [:] - m.city = data.cityname - m.region = data.adname - m.name = data.name - m.address = data.address - m.type = data.type - m.code = data.typecode + def mapName = data.name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + def newName = name.replaceAll('[·\\(\\)()]|·', '') + if(newName == mapName){ + def m = [:] + m.city = data.cityname + m.region = data.adname + m.name = data.name + m.address = data.address + m.type = data.type + m.code = data.typecode - def location = data.location - m.lng = location.lon - m.lat = location.lat - rets << m + def location = data.location + m.lng = location.lon + m.lat = location.lat + rets << m + } }) return rets; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy index 960b75f38..fad0c6bd1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy @@ -33,7 +33,16 @@ class CityStat { // task.run6() // task.run7() // task.run8() - task.dealPriceStat() +// task.dealPriceStat() + task.countDist() + } + def countDist(){ + ExcelUtils.modify("D:\\数据\\其它\\小区统计.xlsx", { d -> + if(d['B']){ + def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city", "region", "name"]) + d['D'] = districts.size() + } + },0); } def dealPriceStat(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy new file mode 100644 index 000000000..0979849cf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +//郑州银行数据导出 +class ZhengzhouBlankStat { + public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com") + + public static final def DISTRICT_FIELDS_MAP = [ + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' + ] + + static main(args) { + def task = new ZhengzhouBlankStat(); + task.run() + } + + def run(){ + def citys = ["郑州","洛阳","平顶山","安阳","许昌","南阳","新乡","焦作","开封","信阳","驻马店","漯河","鹤壁","周口","濮阳","三门峡","商丘","济源"] + //statRegionDistrictCount(citys) +// statCityProcent(citys) + statCityHouseloudongProcent(citys) + + } + + def statRegionDistrictCount(citys){ + def rets = [] + rets << ["城市", "区县", "楼盘数量"] + for(def c : citys){ + def regions = CommonDatautils.getCityRegions(c) + + for(def r : regions){ + def count = CommonDatautils.findRegionDistrictCount("district", c, r) + def ret = [] + ret << c + ret << r + ret << count + rets << ret + } + + } + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省区县小区量统计") + } + + def statCityProcent(citys){ + def rets = [] + rets << ["城市","数据量","地址","地址别名","别名","楼栋总数","户数","产权年限","产权性质","建筑结构","竣工时间","小区当前均价"] + for(def c : citys){ + def dbo = CommonDatautils.findCityInfo(c) + if(dbo){ + def detail = dbo.district_detail + def ret = [] + ret << c + if(detail){ + ret << dbo.existing_district_count//数据量 + ret << detail.address*100// 地址 + ret << detail.addrAlias*100// 地址别名 + ret << detail.alias*100// 别名 + ret << detail.loudongCount*100// 楼栋总数 + ret << detail.householdCount*100// 户数 + ret << detail.propertyLife*100// 产权年限 + ret << detail.houseProperty*100// 产权性质 + ret << detail.buildStructure*100// 建筑结构(按材料划分) + ret << detail.finishBuildingDate*100// 竣工时间 + ret << dbo.price_coverage*100// 小区当前均价 + } + rets << ret + } + } + + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省城市字段覆盖率统计") + } + + def statCityHouseloudongProcent(citys){ + def rets = [] + + rets << ["城市","楼栋","单元","所在层数","房号","面积","套型","楼层"] + for(def c : citys){ + def dbo = CommonDatautils.findCityInfo(c) + if(dbo){ + def detail = dbo.house_loudong_detail + def ret = [] + ret << c + if(detail){ + ret << detail.house_num_rate*100//楼栋 + ret << detail.unit_num_rate*100// 单元 + ret << detail.room_num_rate*100// 所在层数 + ret << detail.room_num_rate*100// 房号 + ret << detail.area_rate*100// 面积 + ret << detail.room_info_rate*100// 套型 + } + def loudong_detail = dbo.loudong_detail + if(loudong_detail){ + ret << loudong_detail.loudong_height_rate*100// 楼层 + } + rets << ret + } + } + + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省房产信息字段覆盖率统计") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy new file mode 100644 index 000000000..a2ec7c80b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil + +class DevProcess { + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + static main(args) { + def task = new DevProcess(); + task.run() + } + def run(){ + def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls") + def index = 0 + + def mcaDistMap = CommonDatautils.queryDistNameCity() + for(def r : rets){ + index ++ + if(index >2){ + def udata = new BasicDBObject() + JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(null, r['E']); + if (jsonObject.get("district") && jsonObject.get("district") == "null") { + jsonObject = FangjiaClientUtils.formatFivelevel(null, r['C']+r['E']); + } + if (jsonObject.get("district") && jsonObject.get("district") == "null") { + jsonObject = FangjiaClientUtils.formatFivelevel(null, r['B']+r['C']+r['E']); + } + + udata.address = r['E'] + udata.create_time = new Date() + udata.industry = r['F'] + udata.level = "国家级" + udata.name = r['D'] + udata.province = r['B'] + udata.type = "农村创业园区" + + if(jsonObject) { + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + udata.city = jsonObject.get("city") + } + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + udata.district = jsonObject.get("district") + } + if(!udata.district && udata.city){ + def city = RegionData.formatCity(udata.city) + def region = RegionData.formatRegion(udata.city, r['C']) + if(region){ + def regionObj = mcaDistMap.get(city+"#"+region) + if(regionObj){ + udata.district = regionObj.district + } + } + } + if(jsonObject.get("township") && jsonObject.get("township")!="null"){ + udata.township = jsonObject.get("township") + } + + if(!udata.district){ + println index+" "+ udata + } + + } + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + + + } + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy new file mode 100644 index 000000000..2a638980a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -0,0 +1,467 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.PgsqlCommUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class GXDdistrictMatch { + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def regionCodeDataMap = PgsqlCommUtils.findDistrictMap(); + + def fullCityMap = CommonDatautils.queryFullNameCity(); + + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + + static main(args) { + def task = new GXDdistrictMatch(); +// task.run() +// task.addOldMatch() +// task.runDb() + +// task.updateDistStatus()//恢复name_cat不正常数据 +// task.saveToDB() //保存新增小区 + task.restoreDist()//恢复小区status=0 + } + + def addOldMatch(){ + //老匹配 + def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + + def runDb(){ + def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + ExcelUtils.modify(fileName, { d -> + if(!d['U']){ + def fullCityObj = fullCityMap.get(d['AD']) + if(fullCityObj){ + def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E']) + def obj = findPoiFromDB(fullCityObj.city, fullRegion, d['C']) + if(obj){ + d['U'] = "地图库" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.type + d['Z'] = obj.address + d['AA'] = obj.lng + d['AB'] = obj.lat + }else{ + obj = findDatas(d['AD'], d['AE'], d['C'], d['C'].replaceAll('[·\\(\\)()]|·', '')) + if(obj){ + d['U'] = "楼盘抓取库" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.property_type + d['Z'] = obj.address + def loc = obj.location + if(loc){ + d['AA'] = loc.x + d['AB'] = loc.y + } + } + } + } + } + }) + } + + def findPoiFromDB(city, region, name){ + def rets = EsSearchUtils.findPois(city, region, name) + def obj = null + if(rets && rets.size()==1){ + obj = rets[0] + } + return obj + } + + + def run(){ + def fileName = "D:\\数据\\国信达\\接口匹配.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['V'] && d['A']){ + def city = getCityFromCityCode(d['A']) + def region = regionCodeDataMap.get(d['A'] + "#" + d['E']) + if (city && region) { + region = RegionData.formatRegion(city, region) + d['AD'] = city + d['AE'] = region + if (region) { + d = match(city, region, d) + } else { + println d['A'] + "#" + d['E'] + " 区域无法格式化" + } + + } else { + println d['A'] + "#" + d['E'] + " 无法格式化" + } + } + }) + } + + def updateDistStatus(){ + def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx" + def index = 0 + ExcelUtils.modify(fileName, { d -> + if(d['U'] && ["名称匹配","名称匹配(Update)","名称在别名中","名称在别名中(Update)","别名匹配"].contains(d['U'])){ + index++ + + def status = d['Z'] + def name_cat = d['AA'] + + println d["V"]+" "+d["X"]+name_cat + + if(status!=null && status.toString().toDouble().toInteger()==0){ + if(name_cat==null){//&& name_cat.toString().toDouble().toInteger()!=0 + + def dist = CommonDatautils.findDistrict(d["V"],d["W"],d["X"], "district") + if(dist){ + def delUdata = new BasicDBObject() + delUdata.put("name_cat", 0) + delUdata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata)) + saveOptLog(dist.city, dist.region, dist.name, "update") + } + } + } + } + }) + } + + def restoreDist(){ + def fileName = "D:\\数据\\国信达\\11城小区需恢复数据.xlsx"//国信达11城(全).xlsx + def index = 0 + ExcelUtils.modify(fileName, { d -> + if(d['U']){ + index++ + def status = d['Z'] + println d["V"]+" "+d["X"] + if(status && status.toString().toDouble().toInteger()==1){ + def dists = CommonDatautils.findAllStatusDistrict(d["V"],d["W"],d["X"], "district") + if(dists && dists.size()==1){ + def dist = dists[0] + def delUdata = new BasicDBObject() + delUdata.put("name_cat", 0) + delUdata.put("status", 0) + delUdata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata)) + saveOptLog(dist.city, dist.region, dist.name, "update-status") + } + } + } + }) + } + + def saveToDB(){ + def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + ExcelUtils.modify(fileName, { d -> + if(d['U'] && d['AG']!='过滤'){ + def udata = new BasicDBObject() + udata.put("city", d['AD']) + udata.put("region", d['AE']) + def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def alias = [] + if(name!=d['C']){ + alias << d['C'] + } + if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ + alias << d['AI'] + } + if(alias){ + udata.put("alias", alias) + } + + udata.put("address", d['AF']) + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(d['AH']){ + type = d['AH'] + }else{ + type = d['Y'] + } + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(d['AA']){ + Long lng = d['AA']*10000000000 + udata.put("lng", lng) + } + if(d['AB']){ + Long lat = d['AB'] * 10000000000 + udata.put("lat", lat) + } + + def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) + def opt = "add" + if(s && s=='成功'){ + saveOptLog(udata.city, udata.region, udata.name, opt) + } + } + }) + + } + + + def match(city, region, d){ + d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "") + if(!d['U']){ + d = matching(city, region, d, d['C'].replace("小区","").replace("号楼","号院").replace("号","号院"), d['D'], "(Update)") + if(!d['U']){ + d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + + if(!d['U']){ + if(d['F'] && d['G']){ + d = matchAddr(city, region, d, "") + } + } + + if(!d['U']){ + d = matchApi(city, region, d, "property", false) + if(!d['U']){ + d = matchApi(city, region, d, null, true) + } + } + + if(!d['U']){ + d = matching2(city, region, d, d['C'], d['D'], "") + } + + if(!d['U']){ + if(d['C'].contains("小区")){ + d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") + }else{ + d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + } + } + return d + } + + def matchApi(city, region, d, cat, useMap){ + def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) + if(!jsonObject){ + jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap) + if(!jsonObject && d['D']){ + jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap) + } + } + if(jsonObject != null && jsonObject.credit>0) { + d['U'] = "接口匹配" + d['V'] = jsonObject.get("city") + d['W'] = jsonObject.get("district") + d['X'] = jsonObject.get("name") + def type = jsonObject.get("category")//propertyType + if(type && type!='null'){ + d['Y'] = type + } + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + } + d['AC'] = jsonObject.get("credit") + if(jsonObject.get("address") && jsonObject.get("address")!='null'){ + d['AH'] = jsonObject.get("address") + } + + } + return d + } + + + def matching(city, region, d, name, alias, str){ + //正常状态下有没有 + def rets = CommonDatautils.findDistrictList(city, region, name, "district") + if(rets){ + if(rets.size()>=1){ + d = setRowValue(rets[0], d, "名称匹配"+str) + } + }else{ + rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()>=1){ + d = setRowValue(rets[0], d, "名称在别名中"+str) + }else{ + rets = CommonDatautils.findDistrictByAddress(city, region, name, "district") + if(rets && rets.size()>=1){ + d = setRowValue(rets[0], d, "名称在地址中"+str) + }else{ + if (d['D']) { + rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district") + if (rets) { + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配"+str) + } + } + } + } + } + } + + if(rets){ + d['AI'] = rets.size() + } + return d; + } + + def matching2(city, region, d, name, alias, str){ + def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的 + if(rets){ + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "名称匹配(非正常)"+str) + } + }else{ + + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district") + if (rets && rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + }else{ + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district") + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + } + } + } + + if(rets){ + d['AI'] = rets.size() + } + return d; + } + + def matchAddr(city, region, d, str){ + def rets = CommonDatautils.findDistrictByAddress(city, region, d['F']+d['G'],"district") + if(rets){ + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "地址匹配") + } + } + if(rets){ + d['AI'] = rets.size() + } + return d + } + + def setRowValue(dbo, d, remark){ + if (dbo) { + d['V'] = dbo.city + d['W'] = dbo.region + d['X'] = dbo.name + d['Y'] = dbo.estate_type2 + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + d['U'] = remark + d['AH'] = dbo.address + } + return d; + } + + def getCityFromCityCode(cityCode){ + def city = null + for(def m : cityCodeDataMap){ + if(m.value == cityCode){ + city = m.key + break; + } + } + return city; + } + + def filterSpacialChar(def s){ + def pattern = ~/[()!;\·]\[【】()]/ + def matcher = pattern.matcher(s) + + def count = matcher.getCount() + if(count && count>0){ + return matcher.replaceAll('_') + } + return null; + } + + + def findDatas(def city, def region, def name1, def name2){ + def sql = "select * from community where city='${city}' and region='${region}' and (name='${name1}' or name='${name2}')" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + println d.property_type + }) + + if(rets){ + def obj = null + for(def r : rets){ + if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){ + obj = r + break + } + } + if(obj){ + return obj; + }else{ + return rets[0]; + } + + } + return null; + } + + def saveOptLog(city, region, name, type){ + def obj = new BasicDBObject(); + obj.put("city",city); + obj.put("region",region); + obj.put("district_name",name); + obj.put("category","district"); + obj.put("c_date", new Date()) + obj.put("m_date", new Date()) + obj.put("type", type) + obj.put("user_name", "高娟") + obj.put("origin", "script") + OPERATION_LOG_COLL.save(obj) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy new file mode 100644 index 000000000..7c9472e42 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -0,0 +1,13 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class TradeDatasDistBatch { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + static main(args) { +// def + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 3abfbaa22..260ef5cb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -228,9 +228,9 @@ class FangjiaClientUtils { def url = null; try { address = address.replace("{","").replace("\\","") - url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" -// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" - + url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") } @@ -314,13 +314,13 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") +// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) - println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy deleted file mode 100644 index 85b3a9a65..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy +++ /dev/null @@ -1,138 +0,0 @@ -package com.fangjia.script.gj.district - -import com.fangjia.core.RegionData -import com.fangjia.script.ccbevaluation.CcbCommonUtils -import com.fangjia.script.ccbevaluation.PgsqlCommUtils -import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.ExcelUtils - -class GXDdistrictMatch { - - static def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - static def regionCodeDataMap = PgsqlCommUtils.findDistrictMap() - - static main(args) { - def task = new GXDdistrictMatch(); - task.run() - } - - def run(){ - def fileName = "D:\\数据\\国信达\\国信达11城2.xlsx" - ExcelUtils.multiThreadingModify(fileName, { d -> - if(!d['V']){ - def city = getCityFromCityCode(d['A']) - def region = regionCodeDataMap.get(d['A'] + "#" + d['E']) - if (city && region) { - region = RegionData.formatRegion(city, region) - if (region) { - d = match(city, region, d) - } else { - println d['A'] + "#" + d['E'] + " 区域无法格式化" - } - } else { - println d['A'] + "#" + d['E'] + " 无法格式化" - } - } - }) - } - - - def match(city, region, d){ - d = matching(city, region, d, d['C'], d['D'], "") - if(!d['U']){ - if(d['C'].contains("小区")){ - d = matching(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") - }else{ - d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)") - } - if(!d['U']){ - d = matching2(city, region, d, d['C'], d['D'], "") - } - - if(!d['U']){ - if(d['C'].contains("小区")){ - d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") - }else{ - d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)") - } - } - - } - } - - - def matching(city, region, d, name, alias, str){ - //正常状态下有没有 - def rets = CommonDatautils.findDistrictList(city, region, name, "district") - if(rets){ - if(rets.size()==1){ - d = setRowValue(rets[0], d, "名称匹配"+str) - }else{ - d['U'] = "名称在多个小区中"+str - } - }else{ - rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") - if(rets && rets.size()==1){ - d = setRowValue(rets[0], d, "名称在别名中"+str) - }else{ - if (d['D']) { - rets = CommonDatautils.findDistrictByAlias(city, region, alias, "district") - if (rets) { - if (rets.size() == 1) { - d = setRowValue(rets[0], d, "别名匹配"+str) - } else { - d['U'] = "别名在多个小区中"+str - } - } - } - } - } - - return d; - } - - def matching2(city, region, d, name, alias, str){ - def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的 - if(rets){ - if (rets.size() == 1) { - d = setRowValue(rets[0], d, "名称匹配(非正常)"+str) - } - }else{ - - rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district") - if (rets && rets.size() == 1) { - d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) - }else{ - rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district") - if (rets.size() == 1) { - d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) - } - } - } - return d; - } - - def setRowValue(dbo, d, remark){ - if (dbo) { - d['V'] = dbo.city - d['W'] = dbo.region - d['X'] = dbo.name - d['Y'] = dbo.estate_type2 - d['Z'] = dbo.status - d['AA'] = dbo.name_cat - d['U'] = remark - } - return d; - } - - def getCityFromCityCode(cityCode){ - def city = null - for(def m : cityCodeDataMap){ - if(m.value == cityCode){ - city = m.key - break; - } - } - return city; - } -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy index d259d858b..674198bd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -23,7 +23,7 @@ class EsSelectTest { def test(){ - def ret = CommonDatautils.findLogs("1007703", "/property/evaluate", "2024-03-13", "2024-03-16") + def ret = CommonDatautils.findLogs("1007741", "/property/evaluate", "2024-03-01", "2024-03-28") } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index c153747fc..dade0fa1f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -5,13 +5,22 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.HttpClient import net.sf.json.JSONObject +import okhttp3.*; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.concurrent.TimeUnit; + + +import okhttp3.Request.Builder; class QiliangTest { - static main(args) { - def task = new QiliangTest() - task.run() - } +// static main(args) { +// def task = new QiliangTest() +//// task.run() +// task.getHttpResult("扬州扬子江北路399号11041") +// } def run(){//地址测试-2024.3.4-结果.xlsx ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> @@ -80,7 +89,7 @@ class QiliangTest { def getHttpResult(address){ def data = new JSONObject() data.put("address", address) - data.put("type","1") + data.put("type","2") Map headers = new HashMap(); headers.put("Accept", "application/json, text/plain, */*"); @@ -104,4 +113,79 @@ class QiliangTest { } return null; } + + + final static String proxyIp = "tunnel2.qg.net"; + final static Integer proxyPort = 14186; + final static String authKey = "113BBC06"; + final static String password = "3B2FBD05B0AF"; + public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8"); + +// public String request(params) throws IOException { +// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort)); +// OkHttpClient client = new OkHttpClient.Builder() +// .proxy(proxy) +// .proxyAuthenticator((route, response){ +// String credential = Credentials.basic(authKey, password); +// return response.request().newBuilder().header("Proxy-Authorization", credential).build(); +// }). +// build(); +//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build(); +// +// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params)); +// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build(); +// +// Response response = client.newCall(request).execute(); +// return response.body().string(); +// +//// return client.newCall(request).execute(); +// } + public static void main(String[] args) { + def proxyHost = "tunnel2.qg.net" + def proxyPort = 14186 + def proxyUsername = '113BBC06' + def proxyPassword = '3B2FBD05B0AF' + + def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign' + def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2'] + + +// 创建代理服务器的连接 + def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}") + +// 打开代理连接 + def proxyConnection = proxy.openConnection() + +// 创建目标URL的连接 + def url = new URL(targetUrl) + def connection = url.openConnection(proxy) + +// 设置请求方法 + connection.setRequestMethod("GET") + +// 添加请求头(可选) + connection.setRequestProperty("Content-Type", "application/json") + +// 构建查询参数 + def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&') + +// 将查询参数附加到目标URL + def finalUrl = "${targetUrl}?${queryString}" + +// 发送请求 + connection.connect() + +// 获取响应 + def responseCode = connection.getResponseCode() + def responseMessage = connection.getResponseMessage() + +// 读取响应内容 + def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) + def responseContent = reader.readLine() + reader.close() + + println "Response Code: ${responseCode}" + println "Response Message: ${responseMessage}" + println "Response Content: ${responseContent}" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index d6ef502f2..50882747b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 3 + def nThread = Runtime.getRuntime().availableProcessors() * 4 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0)