From f56882ae4bac09098b3248150071ba285adc5a02 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Dec 2021 17:16:53 +0800 Subject: [PATCH] update --- .../java/com/fangjia/core/util/ExcelUtil.java | 2 +- projects/fangjia-script/pom.xml | 8 +- .../ccbevaluation/BigdataBuildingBatch.groovy | 3 +- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../BigdataCommunityBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 172 ++++----- .../ccbevaluation/data/CcbDistrictToDB.groovy | 137 +++++++ .../ccbevaluation/data/DistrictMarge.groovy | 150 ++++++++ .../ccbevaluation/data/MatchMapData.groovy | 333 ++++++++++++++++++ .../ronghe/BigdataCommunityGJRHBatch.groovy | 2 +- .../ronghe/ExportDistrictRongHe.groovy | 23 +- .../script/commData/CommonDatautils.groovy | 33 +- .../com/fangjia/script/gj/SynchSeaweed.groovy | 45 +++ .../district/SeaweedDataMatchProcess.groovy | 1 + .../script/gj/es/EsHouseMatchDistrict.groovy | 19 +- .../script/linxsh/util/AddressUtil.groovy | 3 + .../script/thirdPriceCrawl/DataStat.groovy | 2 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 7 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- .../ThirdDataToGatherBatch.groovy | 1 - 20 files changed, 826 insertions(+), 121 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java index a24864225..509490d97 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java @@ -58,7 +58,7 @@ public class ExcelUtil { row.setHeightInPoints(35); Font font = workBook.createFont(); font.setFontName("宋体"); - font.setBoldweight(Font.BOLDWEIGHT_BOLD); + // font.setBoldweight(Font.BOLDWEIGHT_BOLD); font.setFontHeightInPoints((short) 12); CellStyle cellStyle = workBook.createCellStyle(); cellStyle.setFont(font); diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index bcebb2b2b..c71006bb8 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -366,14 +366,14 @@ org.apache.poi poi - 3.17 - + + 3.10-FINAL org.apache.poi poi-ooxml - 3.17 - + + 3.10-FINAL xml-apis diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index a9ea036ef..29e3ac2a4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -22,7 +22,8 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + //def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"] def rows = [] def headers = [] for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 172aa4c7a..b59fe435b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -41,7 +41,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ rets.addAll(rows) println city +" "+rows.size() } - def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false); println "txt 生成" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 6e470ce2c..6241cc73b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]// CcbCommonUtils.citys; def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 6aabf6064..0564c1f41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -267,18 +267,20 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) - -ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\分期数据删除_3144.xlsx",{d-> - if(d['A']){ - def udata = new BasicDBObject() - udata.put("name_cat", 9) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", new ObjectId(d['A'])) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - } -},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\第四批需要删除的数据.xlsx",{d-> +// if(d['A']){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", new ObjectId(d['A'])) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// println index +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> @@ -298,79 +300,79 @@ ExcelUtils.multiThreadingModify("D:\\ //},0) -//def index = 0 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d-> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// -// def alias = [] -// if(d['H'] && d['H']!=dist.name){ -// alias << d['H'] -// if(d['H'].contains("·")){ -// alias << d['H'].replace("·", "") -// } -// if(dist.alias){ -// for(def aliasName : dist.alias){ -// if(!alias.contains(aliasName)){ -// alias << aliasName -// } -// } -// } -// } -// -// def address = null -// if(d['I']){ -// address = AddressUtil.formatAddress(dist.city, d['I']) -// } -// -// Long lng = null -// Long lat = null -// Long lat2 = null -// Long lng2 = null -// if(d['J']){ -// lng = d['J'].toString().toDouble() * 10000000000l; -// lat = d['K'].toString().toDouble() * 10000000000l; -// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) -// if(json != null && json.containsKey("x")) { -// lat2 = json.get("y") * 10000000000l -// lng2 = json.get("x") * 10000000000l -// } -// } -// -// def region = null -// if(dist.region!=d['F']){ -// region = dist.region -// } -// -// def udata = new BasicDBObject() -// if(region){ -// udata.put("region", region) -// } -// -// if(address){ -// udata.put("address", address) -// } -// if(lat2){ -// udata.put("lng", lng) -// udata.put("lat", lat) -// udata.put("lng2", lng2) -// udata.put("lat2", lat2) -// } -// -// if(alias){ -// udata.put("alias", alias) -// } -// if(udata){ -// index++ -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) -// println index+ " "+dist.city +" "+dist.name+" "+udata -// } -// -// } -// } -// -//},0) +def index = 0 +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> + if(d['A']){ + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + + def alias = [] + if(d['H'] && d['H']!=dist.name){ + alias << d['H'] + if(d['H'].contains("·")){ + alias << d['H'].replace("·", "") + } + if(dist.alias){ + for(def aliasName : dist.alias){ + if(!alias.contains(aliasName)){ + alias << aliasName + } + } + } + } + + def address = null + if(d['I']){ + address = AddressUtil.formatAddress(dist.city, d['I']) + } + + Long lng = null + Long lat = null + Long lat2 = null + Long lng2 = null + if(d['J']){ + lng = d['J'].toString().toDouble() * 10000000000l; + lat = d['K'].toString().toDouble() * 10000000000l; + def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) + if(json != null && json.containsKey("x")) { + lat2 = json.get("y") * 10000000000l + lng2 = json.get("x") * 10000000000l + } + } + + def region = null + if(dist.region!=d['F']){ + region = dist.region + } + + def udata = new BasicDBObject() + if(region){ + udata.put("region", region) + } + + if(address){ + udata.put("address", address) + } + if(lat2){ + udata.put("lng", lng) + udata.put("lat", lat) + udata.put("lng2", lng2) + udata.put("lat2", lat2) + } + + if(alias){ + udata.put("alias", alias) + } + if(udata){ + index++ + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println index+ " "+dist.city +" "+dist.name+" "+udata + } + + } + } + +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy new file mode 100644 index 000000000..6ee9f1f56 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -0,0 +1,137 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.mongodb.BasicDBObject + +class CcbDistrictToDB { + + static main(args) { + def task = new CcbDistrictToDB(); + task.run() + } + + def run(){ + def files = ["北京融合匹配_1215.xlsx","广州融合匹配_1215.xlsx","杭州融合匹配_1216.xlsx","合肥融合匹配_1216.xlsx","深圳融合匹配_1215.xlsx","石家庄融合匹配_1216.xlsx","苏州融合匹配_1216.xlsx","武汉融合匹配_1216.xlsx","西安融合匹配_1216.xlsx","重庆融合匹配_1215.xlsx"] + def index = 0 + for(def f : files){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\融合\\10个城市新增小区\\" + f, { d -> +// if(d['I']=="棠德花苑新社区"){ + def city = d['A'] + def region = d['C'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + + def name = "" + def alias = [] + if(d['I']){ + name = nameFileter(d['I']) + if(name != d['I']){ + alias << d['I'] + } + } + if(name){ + if(d['D']!=name && !alias.contains(d['D'])){ + alias << d['D'] + } + }else{ + name = d['D'] + } + + def formatAddress = null + if(d['J']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['J']) + } + if(!formatAddress && d['E']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['E']) + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['K'].toString().toDouble() * 10000000000L + glng = d['L'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['K'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + }else{ + if(d['F']){ + def loactionMap = GeoUtils.gps84ToGcj02(d['F'], d['G']) + def lat = null + def lng = null + if(loactionMap){ + lng = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + lat = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + } + def LatLon = GeoHelper.GetLatLon(lat, lng, 0.01, 180); + def distance = GeoHelper.GetDistance(lat, lng, LatLon.Lat, LatLon.Lon) + + glat = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lat, 6) * 10000000000L + glng = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lon, 6) * 10000000000L + def baidu = BaiDuApiUtils.geoconv(LatLon.Lon, LatLon.Lat, 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + if(name){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", d['A']) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + if(alias){ + data.put("alias",alias) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + } + } + } +// } + },0); + } + } + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + return name; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy new file mode 100644 index 000000000..88156feec --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -0,0 +1,150 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId +import com.fangjia.core.util.BaiDuApiUtils + +class DistrictMarge { + static main(args) { + def task = new DistrictMarge() + task.run() + } + + def run(){ + def rets = getDatas() + def deleteIndex = 0 + def updateIndex = 0 + rets.each{amap_id, valus-> + def updateObj = null + def addAlias = [] + String formatAddress = null + Long lat2 = null + Long lng2 = null + Long lat = null + Long lng = null + def amapName = null + for(def i=0;i + Collections.sort(v,new Comparator(){ + public int compare(def o1, def o2) { + + if(o1['D'].toString().length() > o2['D'].toString().length()){ + return 1; + } + if(o1['D'].toString().length() == o2['D'].toString().length()){ + return 0; + } + return -1; + } + }) + } + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy new file mode 100644 index 000000000..e5ec16dfb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -0,0 +1,333 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.poi.ss.util.RegionUtil +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class MatchMapData { + + def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new MatchMapData() +// task.run() +// task.run1() +// task.run2() + task.run3() +// task.exportHouse() + + } + + def run3(){ + + def files = ["合肥融合匹配_1215.xlsx"] + + for(def fn : files){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合匹配房源\\"+fn,{d-> + def city = d['A'] + def region = d['D'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def count = findHouses2(city, newRegion, "sell", 20160101, d['E']); + if(count){ + d['O'] = "有" + }else{ + count = findThirdParty(city, newRegion, d['E']); + if(count){ + d['O'] = "有" + }else{ + count = findThirdParty(city, region, d['E']); + if(count){ + d['O'] = "有" + } + } + } + } + },0) + } + } + + def run2(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + println dist.name + },0) + } + + def run1(){ + //"合肥","北京","广州","杭州","上海","深圳","石家庄","天津","武汉", + def files = ["苏州"] + for(def f : files){ + def city = f + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d-> + if(d['E']){ + def region = d['E'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def rMap = match(city, newRegion, d) + if(rMap){ + def priceObj = CommonDatautils.findDistrictPrice(rMap.city, rMap.region, rMap.name) + d['M'] = rMap.region + d['N'] = rMap.name + d['O'] = rMap.address + if(priceObj){ + d['P'] = priceObj.avg_price + } + + def dist = CommonDatautils.findDistrictById(rMap.id) + if(dist){ + d['Q'] = dist.lng/10000000000.0 + d['R'] = dist.lat/10000000000.0 + def count = findHouses(dist.city, dist.region, "rent", 20210801, 20210901, dist.name, dist.alias); + if(count){ + d['V'] = "有" + } + } + + if(d['I']){ + def loactionMap = GeoUtils.gps84ToGcj02(d['H'], d['I']) + if(loactionMap){ + d['S'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + d['T'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + } + + def distance = GeoHelper.GetDistance(d['R'], d['Q'], d['T'], d['S']) + d['U'] = distance + } + + + } + } + } + + },0) + } + } + + def match(city, region, d){ + def rMap = SeaweedDataMatchProcess.doMatch(city, region, d['F'], null); + if(!rMap && d['G']){ + def addrFlag = d['G'].toString().matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(addrFlag){ + rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); + } + } + return rMap; + } + + def run(){ + + def files = ["北京","广州","杭州","合肥","上海","深圳","石家庄","天津","武汉","西安","重庆"] + for(def f : files){ + def city = f + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d-> + if(d['E']){ + def region = d['E'] + def newRegion = RegionData.formatRegion(city, region) + + if(newRegion){ + def dist = CommonDatautils.findDistrict(city, newRegion, d['F'],"district") + if(dist){ + d['K']="完全一致" + d['L'] = dist.name_cat + }else{ + dist = CommonDatautils.findSeaweedByMappingAlias(city, newRegion, d['F'],"district") + if(dist){ + d['K']="映射别名" + d['L'] = dist.name_cat + }else{ + dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district") + if(dist){ + d['K']="别名" + } + } + } + }else{ + d['M'] = "区域无法标准化" + } + } + + },0) + } + + } + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + return houses.size(); + } + + + + def findHouses2(def city, def region, def cat, def startDate, def name){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + def alias = [] + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + if(name.contains("号小区")){ + def newName = name.replace("号小区","号") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + println "[房源] "+city +" "+region+" "+ name +" "+houses.size() + return houses.size(); + } + + + def findThirdParty(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + def count = thirdPartyColl.count(q) + println "[第三方] "+city +" "+region+" "+ name +" "+count + return count + } + + def exportHouse(){ + def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"] + for(def f : files) { + def city = f + + def rets = [] + rets << ["建行小区id", "房价网小区id","城市","区域","楼盘名称","单价","月租金","面积","室","厅","卫","出租方式","朝向","楼屋","装修","发布时间"] + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\" + f + ".xlsx", { d -> + def flag = d['K'] + if(flag && flag =="完全一致"){ + def obj = CommonDatautils.findDistrict(city, d['M'], d['N'], "district") + if(obj){ + def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20210901, d['N'], null); + if(houses){ + rets.addAll(houses) + } + } + + } + },0) + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房\\"+city+"_rent.xlsx", rets, "数据") + } + } + + + + def findHouses3(def commid, def fjwid, def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def rets = [] + for(def r : houses){ + rets << [commid, fjwid, r.city,r.region,r.district_name, r.avg_price, r.total_price, r.area,r.room, r.hall,r.toilet,r.rent_type, r.direction,r.floor_info,r.fitment, r.p_date ] + } + + if(rets){ + println city+" "+ name +" "+rets.size() + } + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 87e8f37fa..599442639 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 40b3605c1..596a0c6a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -24,7 +24,8 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { - def citys = CcbCommonUtils.citys; +// def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // exportAllBigdataDealFJW() @@ -34,17 +35,17 @@ class ExportDistrictRongHe extends BaseBatch{ //// println city +" "+cityCode //// } exportBigdataCommunityGJRH(cityCode) -// exportBigdataBuildingFJW(cityCode) -// exportBigdataCommfeeGj(cityCode) + exportBigdataBuildingFJW(cityCode) + exportBigdataCommfeeGj(cityCode) // -// exportBigdataDealFJW(cityCode) -// exportBigdataCommbuildstruGJ(cityCode) -// exportBigdataCommhousetypeGJ(cityCode) -// exportBigdataCommbuildyearGj(cityCode) -// exportBigdataCommhouseuseGJ(cityCode) -// exportBigdataPositionGJ(cityCode) -// exportBigdataDistrictFJW(cityCode) -// exportBigdataAliasFJW(cityCode) + exportBigdataDealFJW(cityCode) + exportBigdataCommbuildstruGJ(cityCode) + exportBigdataCommhousetypeGJ(cityCode) + exportBigdataCommbuildyearGj(cityCode) + exportBigdataCommhouseuseGJ(cityCode) + exportBigdataPositionGJ(cityCode) + exportBigdataDistrictFJW(cityCode) + exportBigdataAliasFJW(cityCode) // // exportBigdataBuildprojFjw(cityCode) // exportBigdataCommlayoutpicFjw(cityCode) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 87c6ba7ed..1549f70c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -204,8 +204,6 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "安居小区"); - DSCursor cursor = getDataStoreConnection().find(q); @@ -1171,6 +1169,37 @@ class CommonDatautils { return ret; } + + static def findSeaweedByMappingAlias(def city, def region, def name, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + + BasicDBObject query = new BasicDBObject(); + query.putAt(QueryOperators.IN, [name]); + q.put("mapping_alias", query); + + def obj = SEAWEED_COLL.findOne(q); + return obj; + } + + static def findAllSeaweed(def cat , def city){ + DBObject q = new BasicDBObject(); + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + def dbcursor = SEAWEED_COLL.find(q).batchSize(5000) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } static def findSeaweedByAddress(def city, def region, def address, def cat){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy new file mode 100644 index 000000000..62e182a5b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class SynchSeaweed { + + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + + public static final DBCollection SEAWEED_ONLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed", mongoDbUrl) + + public static final DBCollection SEAWEED_OFFLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed_1215", offlineUrl) + + static main(args) { + def task = new SynchSeaweed(); + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity(); + for(def city : citys){ + def datas = CommonDatautils.findAllSeaweed("district", city) + saveDatas(datas); + } + } + + def saveDatas(def datas){ + def index = 0 + def ret = [] + for(def d : datas){ + index++; + ret << d + if(ret.size()%10000==0){ + println "第 "+(index/10000)+" 个"+ret.size() + SEAWEED_OFFLINE_COLL.insert(ret); + ret = []; + } + } + if(ret){ + SEAWEED_OFFLINE_COLL.insert(ret) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 9219fc438..319e44ef4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -29,6 +29,7 @@ class SeaweedDataMatchProcess { rMap.lat = obj.lat/10000000000.0 rMap.lng = obj.lng/10000000000.0 rMap.id = obj._id.toString() + rMap.address = obj.address } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy index 2c4adeffd..8d353b9b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy @@ -25,7 +25,7 @@ import java.util.zip.GZIPInputStream class EsHouseMatchDistrict { def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) static main(args) { def task = new EsHouseMatchDistrict() @@ -33,9 +33,9 @@ class EsHouseMatchDistrict { } def houseRun(){ - def citys = ["苏州","杭州"]//CommonUtils.findAllCity(); + def citys = ["杭州"]//CommonUtils.findAllCity(); for(def city : citys){ - def endDate = DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()) + def endDate = DateUtils.parseFromTightByPattern("2021-06-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()) def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1) def houses = findHouseCounts(city, startDate, endDate); match(city, houses); @@ -59,8 +59,9 @@ class EsHouseMatchDistrict { if(rMap){ successCount.addAndGet(1) updateHouseMatchDistrictId(t.id, rMap.id) + println successCount.get()+" "+t.id+" "+ rMap } - println count.get()+" "+t.id+" "+ rMap + } }catch(Exception e){ e.printStackTrace() @@ -82,7 +83,7 @@ class EsHouseMatchDistrict { } def findHouseCounts(def city, def startDate, def endDate){ - QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder date_range = QueryBuilders.rangeQuery("publish_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) .must(QueryBuilders.termQuery("cat", "sell")) @@ -94,13 +95,14 @@ class EsHouseMatchDistrict { // queryBuilder.must(inBuilder) def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + println houses.size() def rets = [] - for(def r : rets){ + for(def r : houses){ if(!r.match_seaweed_id){ rets << r } } - return houses + return rets } @@ -122,7 +124,8 @@ class EsHouseMatchDistrict { } """ HttpRequest request = new HttpRequest() - request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println s } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy index c14e1d3ba..517b6b634 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy @@ -120,6 +120,9 @@ class AddressUtil { if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')) { return formatAddress } + if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷)')) { + return formatAddress + } return null } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 751d405bb..6566d5a16 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,7 +40,7 @@ class DataStat { def run(){ - def source = "我爱我家"// + def source = "城市房产"// def citys = findCrawlCity(source, 202110); for(def city : citys){ def count1 = findCount(source, city, 202110); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 83adfa245..3317161c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -37,8 +37,9 @@ class Esf58Crawl extends BaseBatch{ def static final distMap = [:] def static useDistKeys = [] static def totalCount = new AtomicLong() - //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", - def cityList = [ "上海":"sh|2","北京":"bj|1","天津":"tj|18","重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79", + //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", + //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4", + def cityList = ["东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", "青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", @@ -82,7 +83,7 @@ class Esf58Crawl extends BaseBatch{ "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", - "肇州":"zhaozhou|382", "武汉":"wh|158","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 94f0e3a4c..c106ca013 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["cityhouse.cn":"城市房产"] + def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index ef14c99b0..0b5a49b5d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -40,7 +40,6 @@ class ThirdDataToGatherBatch extends BaseBatch{ static main(args) { runBatch(new ThirdDataToGatherBatch(), args) } - @Override void process(Object args) { println SHORT_DATE