This commit is contained in:
2021-12-20 17:16:53 +08:00
parent 930a54b49c
commit f56882ae4b
20 changed files with 826 additions and 121 deletions
@@ -58,7 +58,7 @@ public class ExcelUtil {
row.setHeightInPoints(35);
Font font = workBook.createFont();
font.setFontName("宋体");
font.setBoldweight(Font.BOLDWEIGHT_BOLD);
// font.setBoldweight(Font.BOLDWEIGHT_BOLD);
font.setFontHeightInPoints((short) 12);
CellStyle cellStyle = workBook.createCellStyle();
cellStyle.setFont(font);
+4 -4
View File
@@ -366,14 +366,14 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.17</version>
<!-- <version>3.10-FINAL</version>-->
<!-- <version>3.17</version>-->
<version>3.10-FINAL</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.17</version>
<!-- <version>3.10-FINAL</version>-->
<!-- <version>3.17</version>-->
<version>3.10-FINAL</version>
<exclusions>
<exclusion>
<groupId>xml-apis</groupId>
@@ -22,7 +22,8 @@ class BigdataBuildingBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
//def citys = CcbCommonUtils.citys;
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]
def rows = []
def headers = []
for (def city : citys) {
@@ -41,7 +41,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
rets.addAll(rows)
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
println "txt 生成"
@@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]// CcbCommonUtils.citys;
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -267,18 +267,20 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
// }
//},0)
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\分期数据删除_3144.xlsx",{d->
if(d['A']){
def udata = new BasicDBObject()
udata.put("name_cat", 9)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['A']))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\第四批需要删除的数据.xlsx",{d->
// if(d['A']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['A']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d->
@@ -298,79 +300,79 @@ ExcelUtils.multiThreadingModify("D:\\
//},0)
//def index = 0
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
//
// def alias = []
// if(d['H'] && d['H']!=dist.name){
// alias << d['H']
// if(d['H'].contains("·")){
// alias << d['H'].replace("·", "")
// }
// if(dist.alias){
// for(def aliasName : dist.alias){
// if(!alias.contains(aliasName)){
// alias << aliasName
// }
// }
// }
// }
//
// def address = null
// if(d['I']){
// address = AddressUtil.formatAddress(dist.city, d['I'])
// }
//
// Long lng = null
// Long lat = null
// Long lat2 = null
// Long lng2 = null
// if(d['J']){
// lng = d['J'].toString().toDouble() * 10000000000l;
// lat = d['K'].toString().toDouble() * 10000000000l;
// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
// if(json != null && json.containsKey("x")) {
// lat2 = json.get("y") * 10000000000l
// lng2 = json.get("x") * 10000000000l
// }
// }
//
// def region = null
// if(dist.region!=d['F']){
// region = dist.region
// }
//
// def udata = new BasicDBObject()
// if(region){
// udata.put("region", region)
// }
//
// if(address){
// udata.put("address", address)
// }
// if(lat2){
// udata.put("lng", lng)
// udata.put("lat", lat)
// udata.put("lng2", lng2)
// udata.put("lat2", lat2)
// }
//
// if(alias){
// udata.put("alias", alias)
// }
// if(udata){
// index++
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// println index+ " "+dist.city +" "+dist.name+" "+udata
// }
//
// }
// }
//
//},0)
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
if(d['A']){
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def alias = []
if(d['H'] && d['H']!=dist.name){
alias << d['H']
if(d['H'].contains("·")){
alias << d['H'].replace("·", "")
}
if(dist.alias){
for(def aliasName : dist.alias){
if(!alias.contains(aliasName)){
alias << aliasName
}
}
}
}
def address = null
if(d['I']){
address = AddressUtil.formatAddress(dist.city, d['I'])
}
Long lng = null
Long lat = null
Long lat2 = null
Long lng2 = null
if(d['J']){
lng = d['J'].toString().toDouble() * 10000000000l;
lat = d['K'].toString().toDouble() * 10000000000l;
def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
if(json != null && json.containsKey("x")) {
lat2 = json.get("y") * 10000000000l
lng2 = json.get("x") * 10000000000l
}
}
def region = null
if(dist.region!=d['F']){
region = dist.region
}
def udata = new BasicDBObject()
if(region){
udata.put("region", region)
}
if(address){
udata.put("address", address)
}
if(lat2){
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
}
if(alias){
udata.put("alias", alias)
}
if(udata){
index++
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println index+ " "+dist.city +" "+dist.name+" "+udata
}
}
}
},0)
@@ -0,0 +1,137 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
class CcbDistrictToDB {
static main(args) {
def task = new CcbDistrictToDB();
task.run()
}
def run(){
def files = ["北京融合匹配_1215.xlsx","广州融合匹配_1215.xlsx","杭州融合匹配_1216.xlsx","合肥融合匹配_1216.xlsx","深圳融合匹配_1215.xlsx","石家庄融合匹配_1216.xlsx","苏州融合匹配_1216.xlsx","武汉融合匹配_1216.xlsx","西安融合匹配_1216.xlsx","重庆融合匹配_1215.xlsx"]
def index = 0
for(def f : files){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\融合\\10个城市新增小区\\" + f, { d ->
// if(d['I']=="棠德花苑新社区"){
def city = d['A']
def region = d['C']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = ""
def alias = []
if(d['I']){
name = nameFileter(d['I'])
if(name != d['I']){
alias << d['I']
}
}
if(name){
if(d['D']!=name && !alias.contains(d['D'])){
alias << d['D']
}
}else{
name = d['D']
}
def formatAddress = null
if(d['J']){
formatAddress = AddressUtil.formatAddress(d['A'], d['J'])
}
if(!formatAddress && d['E']){
formatAddress = AddressUtil.formatAddress(d['A'], d['E'])
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['K'].toString().toDouble() * 10000000000L
glng = d['L'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['K'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}else{
if(d['F']){
def loactionMap = GeoUtils.gps84ToGcj02(d['F'], d['G'])
def lat = null
def lng = null
if(loactionMap){
lng = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
lat = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
}
def LatLon = GeoHelper.GetLatLon(lat, lng, 0.01, 180);
def distance = GeoHelper.GetDistance(lat, lng, LatLon.Lat, LatLon.Lon)
glat = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lat, 6) * 10000000000L
glng = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lon, 6) * 10000000000L
def baidu = BaiDuApiUtils.geoconv(LatLon.Lon, LatLon.Lat, 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
if(name){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", d['A'])
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
if(alias){
data.put("alias",alias)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
}
}
}
// }
},0);
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
}
@@ -0,0 +1,150 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import com.fangjia.core.util.BaiDuApiUtils
class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
task.run()
}
def run(){
def rets = getDatas()
def deleteIndex = 0
def updateIndex = 0
rets.each{amap_id, valus->
def updateObj = null
def addAlias = []
String formatAddress = null
Long lat2 = null
Long lng2 = null
Long lat = null
Long lng = null
def amapName = null
for(def i=0;i<valus.size();i++){
def obj = valus[i]
if(obj['G'] && !formatAddress){
formatAddress = AddressUtil.formatAddress(obj['B'], obj['G'])
}
if(obj['H'] && !lat){
lat = obj['H'].toString().toDouble() * 10000000000L;
lng = obj['I'].toString().toDouble() * 10000000000L;
def json = BaiDuApiUtils.geoconv(obj['I'].toString().toDouble() , obj['H'].toString().toDouble(), 3)
if(json != null && json.containsKey("x")) {
lat2 = json.get("y") * 10000000000L
lng2 = json.get("x") * 10000000000L
}
}
if(obj['F'] && !amapName){
amapName = obj['F']
}
if(i==0){
updateObj = obj
}else{
if(obj['D']){
addAlias << obj['D']
}
//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 8:建行整理删除(单一来源或其它) 9-高德比对王一期不存在删除 10:小区重复合并删除 11:人工整理删除
def udata = new BasicDBObject()
udata.put("name_cat", 10)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(obj['A']));
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
deleteIndex++
println "删除:"+obj['A']
}
}
if(updateObj){
def dist = CommonDatautils.findDistrictById(updateObj['A'])
if(dist){
def alias = dist.alias
if(!alias){
alias = []
}
for(def n : addAlias){
if(!alias.contains(n) && dist.name!=n){
alias << n
}
}
if(amapName && !alias.contains(amapName) && amapName!=dist.name){
alias << amapName
}
def mudata = new BasicDBObject()
if(lat){
mudata.put("lat", lat)
mudata.put("lng", lng)
mudata.put("lat2", lat2)
mudata.put("lng2", lng2)
}
if(formatAddress){
mudata.put("address", formatAddress)
}
if(alias){
mudata.put("alias", alias)
}
if(mudata){
mudata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(updateObj['A']));
DbUtils.seaweed.update(q, new BasicDBObject('$set', mudata))
updateIndex++
println "删除:"+updateObj['A']
}else{
println amap_id+" 此组无更新"
}
}
}
}
println "组:"+rets.size()
println "删:"+deleteIndex
println "修:"+updateIndex
}
def getDatas(){
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\第四批修正的重复数据_1363.xlsx",0,1)
def map = [:]
for(def d : datas){
if(d['E']){
def rets = map.get(d['E'])
if(!rets){
rets = []
map.put(d['E'], rets)
}
rets << d;
}
}
map.each{k,v->
Collections.sort(v,new Comparator<Object>(){
public int compare(def o1, def o2) {
if(o1['D'].toString().length() > o2['D'].toString().length()){
return 1;
}
if(o1['D'].toString().length() == o2['D'].toString().length()){
return 0;
}
return -1;
}
})
}
return map;
}
}
@@ -0,0 +1,333 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class MatchMapData {
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new MatchMapData()
// task.run()
// task.run1()
// task.run2()
task.run3()
// task.exportHouse()
}
def run3(){
def files = ["合肥融合匹配_1215.xlsx"]
for(def fn : files){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合匹配房源\\"+fn,{d->
def city = d['A']
def region = d['D']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def count = findHouses2(city, newRegion, "sell", 20160101, d['E']);
if(count){
d['O'] = "有"
}else{
count = findThirdParty(city, newRegion, d['E']);
if(count){
d['O'] = "有"
}else{
count = findThirdParty(city, region, d['E']);
if(count){
d['O'] = "有"
}
}
}
}
},0)
}
}
def run2(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
println dist.name
},0)
}
def run1(){
//"合肥","北京","广州","杭州","上海","深圳","石家庄","天津","武汉",
def files = ["苏州"]
for(def f : files){
def city = f
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d->
if(d['E']){
def region = d['E']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def rMap = match(city, newRegion, d)
if(rMap){
def priceObj = CommonDatautils.findDistrictPrice(rMap.city, rMap.region, rMap.name)
d['M'] = rMap.region
d['N'] = rMap.name
d['O'] = rMap.address
if(priceObj){
d['P'] = priceObj.avg_price
}
def dist = CommonDatautils.findDistrictById(rMap.id)
if(dist){
d['Q'] = dist.lng/10000000000.0
d['R'] = dist.lat/10000000000.0
def count = findHouses(dist.city, dist.region, "rent", 20210801, 20210901, dist.name, dist.alias);
if(count){
d['V'] = "有"
}
}
if(d['I']){
def loactionMap = GeoUtils.gps84ToGcj02(d['H'], d['I'])
if(loactionMap){
d['S'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
d['T'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
}
def distance = GeoHelper.GetDistance(d['R'], d['Q'], d['T'], d['S'])
d['U'] = distance
}
}
}
}
},0)
}
}
def match(city, region, d){
def rMap = SeaweedDataMatchProcess.doMatch(city, region, d['F'], null);
if(!rMap && d['G']){
def addrFlag = d['G'].toString().matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
}
}
return rMap;
}
def run(){
def files = ["北京","广州","杭州","合肥","上海","深圳","石家庄","天津","武汉","西安","重庆"]
for(def f : files){
def city = f
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d->
if(d['E']){
def region = d['E']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def dist = CommonDatautils.findDistrict(city, newRegion, d['F'],"district")
if(dist){
d['K']="完全一致"
d['L'] = dist.name_cat
}else{
dist = CommonDatautils.findSeaweedByMappingAlias(city, newRegion, d['F'],"district")
if(dist){
d['K']="映射别名"
d['L'] = dist.name_cat
}else{
dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district")
if(dist){
d['K']="别名"
}
}
}
}else{
d['M'] = "区域无法标准化"
}
}
},0)
}
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size();
}
def findHouses2(def city, def region, def cat, def startDate, def name){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
def alias = []
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(name.contains("号小区")){
def newName = name.replace("号小区","号")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
println "[房源] "+city +" "+region+" "+ name +" "+houses.size()
return houses.size();
}
def findThirdParty(def city, def region, def name){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name",name)
def count = thirdPartyColl.count(q)
println "[第三方] "+city +" "+region+" "+ name +" "+count
return count
}
def exportHouse(){
def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"]
for(def f : files) {
def city = f
def rets = []
rets << ["建行小区id", "房价网小区id","城市","区域","楼盘名称","单价","月租金","面积","室","厅","卫","出租方式","朝向","楼屋","装修","发布时间"]
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\" + f + ".xlsx", { d ->
def flag = d['K']
if(flag && flag =="完全一致"){
def obj = CommonDatautils.findDistrict(city, d['M'], d['N'], "district")
if(obj){
def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20210901, d['N'], null);
if(houses){
rets.addAll(houses)
}
}
}
},0)
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房\\"+city+"_rent.xlsx", rets, "数据")
}
}
def findHouses3(def commid, def fjwid, def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def rets = []
for(def r : houses){
rets << [commid, fjwid, r.city,r.region,r.district_name, r.avg_price, r.total_price, r.area,r.room, r.hall,r.toilet,r.rent_type, r.direction,r.floor_info,r.fitment, r.p_date ]
}
if(rets){
println city+" "+ name +" "+rets.size()
}
return rets;
}
}
@@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys;
def rows = []
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) {
@@ -24,7 +24,8 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def citys = CcbCommonUtils.citys;
// def citys = CcbCommonUtils.citys;
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// exportAllBigdataDealFJW()
@@ -34,17 +35,17 @@ class ExportDistrictRongHe extends BaseBatch{
//// println city +" "+cityCode
//// }
exportBigdataCommunityGJRH(cityCode)
// exportBigdataBuildingFJW(cityCode)
// exportBigdataCommfeeGj(cityCode)
exportBigdataBuildingFJW(cityCode)
exportBigdataCommfeeGj(cityCode)
//
// exportBigdataDealFJW(cityCode)
// exportBigdataCommbuildstruGJ(cityCode)
// exportBigdataCommhousetypeGJ(cityCode)
// exportBigdataCommbuildyearGj(cityCode)
// exportBigdataCommhouseuseGJ(cityCode)
// exportBigdataPositionGJ(cityCode)
// exportBigdataDistrictFJW(cityCode)
// exportBigdataAliasFJW(cityCode)
exportBigdataDealFJW(cityCode)
exportBigdataCommbuildstruGJ(cityCode)
exportBigdataCommhousetypeGJ(cityCode)
exportBigdataCommbuildyearGj(cityCode)
exportBigdataCommhouseuseGJ(cityCode)
exportBigdataPositionGJ(cityCode)
exportBigdataDistrictFJW(cityCode)
exportBigdataAliasFJW(cityCode)
//
// exportBigdataBuildprojFjw(cityCode)
// exportBigdataCommlayoutpicFjw(cityCode)
@@ -204,8 +204,6 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "安居小区");
DSCursor cursor = getDataStoreConnection().find(q);
@@ -1171,6 +1169,37 @@ class CommonDatautils {
return ret;
}
static def findSeaweedByMappingAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('status',0)
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, [name]);
q.put("mapping_alias", query);
def obj = SEAWEED_COLL.findOne(q);
return obj;
}
static def findAllSeaweed(def cat , def city){
DBObject q = new BasicDBObject();
q.put('city',city)
q.put('cat', cat)
q.put('status',0)
def dbcursor = SEAWEED_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
static def findSeaweedByAddress(def city, def region, def address, def cat){
@@ -0,0 +1,45 @@
package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
class SynchSeaweed {
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection SEAWEED_ONLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed", mongoDbUrl)
public static final DBCollection SEAWEED_OFFLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed_1215", offlineUrl)
static main(args) {
def task = new SynchSeaweed();
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity();
for(def city : citys){
def datas = CommonDatautils.findAllSeaweed("district", city)
saveDatas(datas);
}
}
def saveDatas(def datas){
def index = 0
def ret = []
for(def d : datas){
index++;
ret << d
if(ret.size()%10000==0){
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
SEAWEED_OFFLINE_COLL.insert(ret);
ret = [];
}
}
if(ret){
SEAWEED_OFFLINE_COLL.insert(ret)
}
}
}
@@ -29,6 +29,7 @@ class SeaweedDataMatchProcess {
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
rMap.id = obj._id.toString()
rMap.address = obj.address
}
}
}
@@ -25,7 +25,7 @@ import java.util.zip.GZIPInputStream
class EsHouseMatchDistrict {
def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
static main(args) {
def task = new EsHouseMatchDistrict()
@@ -33,9 +33,9 @@ class EsHouseMatchDistrict {
}
def houseRun(){
def citys = ["ËÕÖÝ","º¼ÖÝ"]//CommonUtils.findAllCity();
def citys = ["º¼ÖÝ"]//CommonUtils.findAllCity();
for(def city : citys){
def endDate = DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date())
def endDate = DateUtils.parseFromTightByPattern("2021-06-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date())
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1)
def houses = findHouseCounts(city, startDate, endDate);
match(city, houses);
@@ -59,8 +59,9 @@ class EsHouseMatchDistrict {
if(rMap){
successCount.addAndGet(1)
updateHouseMatchDistrictId(t.id, rMap.id)
println successCount.get()+" "+t.id+" "+ rMap
}
println count.get()+" "+t.id+" "+ rMap
}
}catch(Exception e){
e.printStackTrace()
@@ -82,7 +83,7 @@ class EsHouseMatchDistrict {
}
def findHouseCounts(def city, def startDate, def endDate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder date_range = QueryBuilders.rangeQuery("publish_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("cat", "sell"))
@@ -94,13 +95,14 @@ class EsHouseMatchDistrict {
// queryBuilder.must(inBuilder)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
println houses.size()
def rets = []
for(def r : rets){
for(def r : houses){
if(!r.match_seaweed_id){
rets << r
}
}
return houses
return rets
}
@@ -122,7 +124,8 @@ class EsHouseMatchDistrict {
}
"""
HttpRequest request = new HttpRequest()
request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
println s
}
}
@@ -120,6 +120,9 @@ class AddressUtil {
if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')) {
return formatAddress
}
if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷)')) {
return formatAddress
}
return null
}
@@ -40,7 +40,7 @@ class DataStat {
def run(){
def source = "我爱我家"//
def source = "城市房产"//
def citys = findCrawlCity(source, 202110);
for(def city : citys){
def count1 = findCount(source, city, 202110);
@@ -37,8 +37,9 @@ class Esf58Crawl extends BaseBatch{
def static final distMap = [:]
def static useDistKeys = []
static def totalCount = new AtomicLong()
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
def cityList = [ "上海":"sh|2","北京":"bj|1","天津":"tj|18","重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79",
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18",
//"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4",
def cityList = ["东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
"青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669",
"沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
@@ -82,7 +83,7 @@ class Esf58Crawl extends BaseBatch{
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
"肇州":"zhaozhou|382", "武汉":"wh|158","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
"肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
"黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296",
"咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517",
"仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896",
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
def sourceMap = ["cityhouse.cn":"城市房产"]
def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -40,7 +40,6 @@ class ThirdDataToGatherBatch extends BaseBatch{
static main(args) {
runBatch(new ThirdDataToGatherBatch(), args)
}
@Override
void process(Object args) {
println SHORT_DATE