update
This commit is contained in:
@@ -58,7 +58,7 @@ public class ExcelUtil {
|
||||
row.setHeightInPoints(35);
|
||||
Font font = workBook.createFont();
|
||||
font.setFontName("宋体");
|
||||
font.setBoldweight(Font.BOLDWEIGHT_BOLD);
|
||||
// font.setBoldweight(Font.BOLDWEIGHT_BOLD);
|
||||
font.setFontHeightInPoints((short) 12);
|
||||
CellStyle cellStyle = workBook.createCellStyle();
|
||||
cellStyle.setFont(font);
|
||||
|
||||
@@ -366,14 +366,14 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<version>3.17</version>
|
||||
<!-- <version>3.10-FINAL</version>-->
|
||||
<!-- <version>3.17</version>-->
|
||||
<version>3.10-FINAL</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>3.17</version>
|
||||
<!-- <version>3.10-FINAL</version>-->
|
||||
<!-- <version>3.17</version>-->
|
||||
<version>3.10-FINAL</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>xml-apis</groupId>
|
||||
|
||||
+2
-1
@@ -22,7 +22,8 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
//def citys = CcbCommonUtils.citys;
|
||||
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]
|
||||
def rows = []
|
||||
def headers = []
|
||||
for (def city : citys) {
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]// CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
|
||||
+87
-85
@@ -267,18 +267,20 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\分期数据删除_3144.xlsx",{d->
|
||||
if(d['A']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 9)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['A']))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
},0)
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\第四批需要删除的数据.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['A']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d->
|
||||
@@ -298,79 +300,79 @@ ExcelUtils.multiThreadingModify("D:\\
|
||||
//},0)
|
||||
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
//
|
||||
// def alias = []
|
||||
// if(d['H'] && d['H']!=dist.name){
|
||||
// alias << d['H']
|
||||
// if(d['H'].contains("·")){
|
||||
// alias << d['H'].replace("·", "")
|
||||
// }
|
||||
// if(dist.alias){
|
||||
// for(def aliasName : dist.alias){
|
||||
// if(!alias.contains(aliasName)){
|
||||
// alias << aliasName
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def address = null
|
||||
// if(d['I']){
|
||||
// address = AddressUtil.formatAddress(dist.city, d['I'])
|
||||
// }
|
||||
//
|
||||
// Long lng = null
|
||||
// Long lat = null
|
||||
// Long lat2 = null
|
||||
// Long lng2 = null
|
||||
// if(d['J']){
|
||||
// lng = d['J'].toString().toDouble() * 10000000000l;
|
||||
// lat = d['K'].toString().toDouble() * 10000000000l;
|
||||
// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
|
||||
// if(json != null && json.containsKey("x")) {
|
||||
// lat2 = json.get("y") * 10000000000l
|
||||
// lng2 = json.get("x") * 10000000000l
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def region = null
|
||||
// if(dist.region!=d['F']){
|
||||
// region = dist.region
|
||||
// }
|
||||
//
|
||||
// def udata = new BasicDBObject()
|
||||
// if(region){
|
||||
// udata.put("region", region)
|
||||
// }
|
||||
//
|
||||
// if(address){
|
||||
// udata.put("address", address)
|
||||
// }
|
||||
// if(lat2){
|
||||
// udata.put("lng", lng)
|
||||
// udata.put("lat", lat)
|
||||
// udata.put("lng2", lng2)
|
||||
// udata.put("lat2", lat2)
|
||||
// }
|
||||
//
|
||||
// if(alias){
|
||||
// udata.put("alias", alias)
|
||||
// }
|
||||
// if(udata){
|
||||
// index++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// println index+ " "+dist.city +" "+dist.name+" "+udata
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
|
||||
if(d['A']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
|
||||
def alias = []
|
||||
if(d['H'] && d['H']!=dist.name){
|
||||
alias << d['H']
|
||||
if(d['H'].contains("·")){
|
||||
alias << d['H'].replace("·", "")
|
||||
}
|
||||
if(dist.alias){
|
||||
for(def aliasName : dist.alias){
|
||||
if(!alias.contains(aliasName)){
|
||||
alias << aliasName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def address = null
|
||||
if(d['I']){
|
||||
address = AddressUtil.formatAddress(dist.city, d['I'])
|
||||
}
|
||||
|
||||
Long lng = null
|
||||
Long lat = null
|
||||
Long lat2 = null
|
||||
Long lng2 = null
|
||||
if(d['J']){
|
||||
lng = d['J'].toString().toDouble() * 10000000000l;
|
||||
lat = d['K'].toString().toDouble() * 10000000000l;
|
||||
def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
|
||||
if(json != null && json.containsKey("x")) {
|
||||
lat2 = json.get("y") * 10000000000l
|
||||
lng2 = json.get("x") * 10000000000l
|
||||
}
|
||||
}
|
||||
|
||||
def region = null
|
||||
if(dist.region!=d['F']){
|
||||
region = dist.region
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
if(region){
|
||||
udata.put("region", region)
|
||||
}
|
||||
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
if(lat2){
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
}
|
||||
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
if(udata){
|
||||
index++
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println index+ " "+dist.city +" "+dist.name+" "+udata
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class CcbDistrictToDB {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistrictToDB();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def files = ["北京融合匹配_1215.xlsx","广州融合匹配_1215.xlsx","杭州融合匹配_1216.xlsx","合肥融合匹配_1216.xlsx","深圳融合匹配_1215.xlsx","石家庄融合匹配_1216.xlsx","苏州融合匹配_1216.xlsx","武汉融合匹配_1216.xlsx","西安融合匹配_1216.xlsx","重庆融合匹配_1215.xlsx"]
|
||||
def index = 0
|
||||
for(def f : files){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\融合\\10个城市新增小区\\" + f, { d ->
|
||||
// if(d['I']=="棠德花苑新社区"){
|
||||
def city = d['A']
|
||||
def region = d['C']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
|
||||
def name = ""
|
||||
def alias = []
|
||||
if(d['I']){
|
||||
name = nameFileter(d['I'])
|
||||
if(name != d['I']){
|
||||
alias << d['I']
|
||||
}
|
||||
}
|
||||
if(name){
|
||||
if(d['D']!=name && !alias.contains(d['D'])){
|
||||
alias << d['D']
|
||||
}
|
||||
}else{
|
||||
name = d['D']
|
||||
}
|
||||
|
||||
def formatAddress = null
|
||||
if(d['J']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['J'])
|
||||
}
|
||||
if(!formatAddress && d['E']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['E'])
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['K'].toString().toDouble() * 10000000000L
|
||||
glng = d['L'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['K'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}else{
|
||||
if(d['F']){
|
||||
def loactionMap = GeoUtils.gps84ToGcj02(d['F'], d['G'])
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(loactionMap){
|
||||
lng = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
lat = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
}
|
||||
def LatLon = GeoHelper.GetLatLon(lat, lng, 0.01, 180);
|
||||
def distance = GeoHelper.GetDistance(lat, lng, LatLon.Lat, LatLon.Lon)
|
||||
|
||||
glat = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lat, 6) * 10000000000L
|
||||
glng = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lon, 6) * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(LatLon.Lon, LatLon.Lat, 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
}
|
||||
}
|
||||
}
|
||||
// }
|
||||
},0);
|
||||
}
|
||||
}
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
class DistrictMarge {
|
||||
static main(args) {
|
||||
def task = new DistrictMarge()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = getDatas()
|
||||
def deleteIndex = 0
|
||||
def updateIndex = 0
|
||||
rets.each{amap_id, valus->
|
||||
def updateObj = null
|
||||
def addAlias = []
|
||||
String formatAddress = null
|
||||
Long lat2 = null
|
||||
Long lng2 = null
|
||||
Long lat = null
|
||||
Long lng = null
|
||||
def amapName = null
|
||||
for(def i=0;i<valus.size();i++){
|
||||
def obj = valus[i]
|
||||
if(obj['G'] && !formatAddress){
|
||||
formatAddress = AddressUtil.formatAddress(obj['B'], obj['G'])
|
||||
}
|
||||
|
||||
if(obj['H'] && !lat){
|
||||
lat = obj['H'].toString().toDouble() * 10000000000L;
|
||||
lng = obj['I'].toString().toDouble() * 10000000000L;
|
||||
def json = BaiDuApiUtils.geoconv(obj['I'].toString().toDouble() , obj['H'].toString().toDouble(), 3)
|
||||
if(json != null && json.containsKey("x")) {
|
||||
lat2 = json.get("y") * 10000000000L
|
||||
lng2 = json.get("x") * 10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
if(obj['F'] && !amapName){
|
||||
amapName = obj['F']
|
||||
}
|
||||
if(i==0){
|
||||
updateObj = obj
|
||||
}else{
|
||||
if(obj['D']){
|
||||
addAlias << obj['D']
|
||||
}
|
||||
//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 8:建行整理删除(单一来源或其它) 9-高德比对王一期不存在删除 10:小区重复合并删除 11:人工整理删除
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 10)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(obj['A']));
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
deleteIndex++
|
||||
println "删除:"+obj['A']
|
||||
}
|
||||
}
|
||||
|
||||
if(updateObj){
|
||||
def dist = CommonDatautils.findDistrictById(updateObj['A'])
|
||||
if(dist){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
for(def n : addAlias){
|
||||
if(!alias.contains(n) && dist.name!=n){
|
||||
alias << n
|
||||
}
|
||||
}
|
||||
if(amapName && !alias.contains(amapName) && amapName!=dist.name){
|
||||
alias << amapName
|
||||
}
|
||||
|
||||
def mudata = new BasicDBObject()
|
||||
if(lat){
|
||||
mudata.put("lat", lat)
|
||||
mudata.put("lng", lng)
|
||||
mudata.put("lat2", lat2)
|
||||
mudata.put("lng2", lng2)
|
||||
}
|
||||
if(formatAddress){
|
||||
mudata.put("address", formatAddress)
|
||||
}
|
||||
if(alias){
|
||||
mudata.put("alias", alias)
|
||||
}
|
||||
|
||||
if(mudata){
|
||||
mudata.put("m_date",new Date())
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(updateObj['A']));
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', mudata))
|
||||
updateIndex++
|
||||
|
||||
println "删除:"+updateObj['A']
|
||||
}else{
|
||||
println amap_id+" 此组无更新"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "组:"+rets.size()
|
||||
println "删:"+deleteIndex
|
||||
println "修:"+updateIndex
|
||||
}
|
||||
|
||||
def getDatas(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\第四批修正的重复数据_1363.xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def rets = map.get(d['E'])
|
||||
if(!rets){
|
||||
rets = []
|
||||
map.put(d['E'], rets)
|
||||
}
|
||||
rets << d;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
map.each{k,v->
|
||||
Collections.sort(v,new Comparator<Object>(){
|
||||
public int compare(def o1, def o2) {
|
||||
|
||||
if(o1['D'].toString().length() > o2['D'].toString().length()){
|
||||
return 1;
|
||||
}
|
||||
if(o1['D'].toString().length() == o2['D'].toString().length()){
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+333
@@ -0,0 +1,333 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class MatchMapData {
|
||||
|
||||
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchMapData()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
// task.exportHouse()
|
||||
|
||||
}
|
||||
|
||||
def run3(){
|
||||
|
||||
def files = ["合肥融合匹配_1215.xlsx"]
|
||||
|
||||
for(def fn : files){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合匹配房源\\"+fn,{d->
|
||||
def city = d['A']
|
||||
def region = d['D']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def count = findHouses2(city, newRegion, "sell", 20160101, d['E']);
|
||||
if(count){
|
||||
d['O'] = "有"
|
||||
}else{
|
||||
count = findThirdParty(city, newRegion, d['E']);
|
||||
if(count){
|
||||
d['O'] = "有"
|
||||
}else{
|
||||
count = findThirdParty(city, region, d['E']);
|
||||
if(count){
|
||||
d['O'] = "有"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
println dist.name
|
||||
},0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
//"合肥","北京","广州","杭州","上海","深圳","石家庄","天津","武汉",
|
||||
def files = ["苏州"]
|
||||
for(def f : files){
|
||||
def city = f
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d->
|
||||
if(d['E']){
|
||||
def region = d['E']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def rMap = match(city, newRegion, d)
|
||||
if(rMap){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(rMap.city, rMap.region, rMap.name)
|
||||
d['M'] = rMap.region
|
||||
d['N'] = rMap.name
|
||||
d['O'] = rMap.address
|
||||
if(priceObj){
|
||||
d['P'] = priceObj.avg_price
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(rMap.id)
|
||||
if(dist){
|
||||
d['Q'] = dist.lng/10000000000.0
|
||||
d['R'] = dist.lat/10000000000.0
|
||||
def count = findHouses(dist.city, dist.region, "rent", 20210801, 20210901, dist.name, dist.alias);
|
||||
if(count){
|
||||
d['V'] = "有"
|
||||
}
|
||||
}
|
||||
|
||||
if(d['I']){
|
||||
def loactionMap = GeoUtils.gps84ToGcj02(d['H'], d['I'])
|
||||
if(loactionMap){
|
||||
d['S'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
d['T'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
}
|
||||
|
||||
def distance = GeoHelper.GetDistance(d['R'], d['Q'], d['T'], d['S'])
|
||||
d['U'] = distance
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
def match(city, region, d){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, region, d['F'], null);
|
||||
if(!rMap && d['G']){
|
||||
def addrFlag = d['G'].toString().matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
def files = ["北京","广州","杭州","合肥","上海","深圳","石家庄","天津","武汉","西安","重庆"]
|
||||
for(def f : files){
|
||||
def city = f
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d->
|
||||
if(d['E']){
|
||||
def region = d['E']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
|
||||
if(newRegion){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, d['F'],"district")
|
||||
if(dist){
|
||||
d['K']="完全一致"
|
||||
d['L'] = dist.name_cat
|
||||
}else{
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(city, newRegion, d['F'],"district")
|
||||
if(dist){
|
||||
d['K']="映射别名"
|
||||
d['L'] = dist.name_cat
|
||||
}else{
|
||||
dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district")
|
||||
if(dist){
|
||||
d['K']="别名"
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['M'] = "区域无法标准化"
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses2(def city, def region, def cat, def startDate, def name){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
def alias = []
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
if(name.contains("号小区")){
|
||||
def newName = name.replace("号小区","号")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
println "[房源] "+city +" "+region+" "+ name +" "+houses.size()
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
def findThirdParty(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("region",region)
|
||||
q.put("name",name)
|
||||
def count = thirdPartyColl.count(q)
|
||||
println "[第三方] "+city +" "+region+" "+ name +" "+count
|
||||
return count
|
||||
}
|
||||
|
||||
def exportHouse(){
|
||||
def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"]
|
||||
for(def f : files) {
|
||||
def city = f
|
||||
|
||||
def rets = []
|
||||
rets << ["建行小区id", "房价网小区id","城市","区域","楼盘名称","单价","月租金","面积","室","厅","卫","出租方式","朝向","楼屋","装修","发布时间"]
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\" + f + ".xlsx", { d ->
|
||||
def flag = d['K']
|
||||
if(flag && flag =="完全一致"){
|
||||
def obj = CommonDatautils.findDistrict(city, d['M'], d['N'], "district")
|
||||
if(obj){
|
||||
def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20210901, d['N'], null);
|
||||
if(houses){
|
||||
rets.addAll(houses)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房\\"+city+"_rent.xlsx", rets, "数据")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses3(def commid, def fjwid, def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def rets = []
|
||||
for(def r : houses){
|
||||
rets << [commid, fjwid, r.city,r.region,r.district_name, r.avg_price, r.total_price, r.area,r.room, r.hall,r.toilet,r.rent_type, r.direction,r.floor_info,r.fitment, r.p_date ]
|
||||
}
|
||||
|
||||
if(rets){
|
||||
println city+" "+ name +" "+rets.size()
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+1
-1
@@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// for (def city : citys) {
|
||||
|
||||
+12
-11
@@ -24,7 +24,8 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
// exportAllBigdataDealFJW()
|
||||
@@ -34,17 +35,17 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
//// println city +" "+cityCode
|
||||
//// }
|
||||
exportBigdataCommunityGJRH(cityCode)
|
||||
// exportBigdataBuildingFJW(cityCode)
|
||||
// exportBigdataCommfeeGj(cityCode)
|
||||
exportBigdataBuildingFJW(cityCode)
|
||||
exportBigdataCommfeeGj(cityCode)
|
||||
//
|
||||
// exportBigdataDealFJW(cityCode)
|
||||
// exportBigdataCommbuildstruGJ(cityCode)
|
||||
// exportBigdataCommhousetypeGJ(cityCode)
|
||||
// exportBigdataCommbuildyearGj(cityCode)
|
||||
// exportBigdataCommhouseuseGJ(cityCode)
|
||||
// exportBigdataPositionGJ(cityCode)
|
||||
// exportBigdataDistrictFJW(cityCode)
|
||||
// exportBigdataAliasFJW(cityCode)
|
||||
exportBigdataDealFJW(cityCode)
|
||||
exportBigdataCommbuildstruGJ(cityCode)
|
||||
exportBigdataCommhousetypeGJ(cityCode)
|
||||
exportBigdataCommbuildyearGj(cityCode)
|
||||
exportBigdataCommhouseuseGJ(cityCode)
|
||||
exportBigdataPositionGJ(cityCode)
|
||||
exportBigdataDistrictFJW(cityCode)
|
||||
exportBigdataAliasFJW(cityCode)
|
||||
//
|
||||
// exportBigdataBuildprojFjw(cityCode)
|
||||
// exportBigdataCommlayoutpicFjw(cityCode)
|
||||
|
||||
+31
-2
@@ -204,8 +204,6 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "安居小区");
|
||||
|
||||
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
@@ -1171,6 +1169,37 @@ class CommonDatautils {
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static def findSeaweedByMappingAlias(def city, def region, def name, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
|
||||
BasicDBObject query = new BasicDBObject();
|
||||
query.putAt(QueryOperators.IN, [name]);
|
||||
q.put("mapping_alias", query);
|
||||
|
||||
def obj = SEAWEED_COLL.findOne(q);
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def findAllSeaweed(def cat , def city){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
def dbcursor = SEAWEED_COLL.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
static def findSeaweedByAddress(def city, def region, def address, def cat){
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class SynchSeaweed {
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
|
||||
public static final DBCollection SEAWEED_ONLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed", mongoDbUrl)
|
||||
|
||||
public static final DBCollection SEAWEED_OFFLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed_1215", offlineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new SynchSeaweed();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def datas = CommonDatautils.findAllSeaweed("district", city)
|
||||
saveDatas(datas);
|
||||
}
|
||||
}
|
||||
|
||||
def saveDatas(def datas){
|
||||
def index = 0
|
||||
def ret = []
|
||||
for(def d : datas){
|
||||
index++;
|
||||
ret << d
|
||||
if(ret.size()%10000==0){
|
||||
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
|
||||
SEAWEED_OFFLINE_COLL.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
if(ret){
|
||||
SEAWEED_OFFLINE_COLL.insert(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
@@ -29,6 +29,7 @@ class SeaweedDataMatchProcess {
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
rMap.id = obj._id.toString()
|
||||
rMap.address = obj.address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+11
-8
@@ -25,7 +25,7 @@ import java.util.zip.GZIPInputStream
|
||||
class EsHouseMatchDistrict {
|
||||
|
||||
def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
|
||||
static main(args) {
|
||||
def task = new EsHouseMatchDistrict()
|
||||
@@ -33,9 +33,9 @@ class EsHouseMatchDistrict {
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
def citys = ["ËÕÖÝ","º¼ÖÝ"]//CommonUtils.findAllCity();
|
||||
def citys = ["º¼ÖÝ"]//CommonUtils.findAllCity();
|
||||
for(def city : citys){
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date())
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-06-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date())
|
||||
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1)
|
||||
def houses = findHouseCounts(city, startDate, endDate);
|
||||
match(city, houses);
|
||||
@@ -59,8 +59,9 @@ class EsHouseMatchDistrict {
|
||||
if(rMap){
|
||||
successCount.addAndGet(1)
|
||||
updateHouseMatchDistrictId(t.id, rMap.id)
|
||||
println successCount.get()+" "+t.id+" "+ rMap
|
||||
}
|
||||
println count.get()+" "+t.id+" "+ rMap
|
||||
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
@@ -82,7 +83,7 @@ class EsHouseMatchDistrict {
|
||||
}
|
||||
|
||||
def findHouseCounts(def city, def startDate, def endDate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("publish_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("cat", "sell"))
|
||||
@@ -94,13 +95,14 @@ class EsHouseMatchDistrict {
|
||||
// queryBuilder.must(inBuilder)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
println houses.size()
|
||||
def rets = []
|
||||
for(def r : rets){
|
||||
for(def r : houses){
|
||||
if(!r.match_seaweed_id){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
return houses
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +124,8 @@ class EsHouseMatchDistrict {
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
println s
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
@@ -120,6 +120,9 @@ class AddressUtil {
|
||||
if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')) {
|
||||
return formatAddress
|
||||
}
|
||||
if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷)')) {
|
||||
return formatAddress
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "我爱我家"//
|
||||
def source = "城市房产"//
|
||||
def citys = findCrawlCity(source, 202110);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202110);
|
||||
|
||||
+4
-3
@@ -37,8 +37,9 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static final distMap = [:]
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
|
||||
def cityList = [ "上海":"sh|2","北京":"bj|1","天津":"tj|18","重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79",
|
||||
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18",
|
||||
//"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4",
|
||||
def cityList = ["东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
|
||||
"青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669",
|
||||
"沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
|
||||
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
|
||||
@@ -82,7 +83,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
|
||||
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
|
||||
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
|
||||
"肇州":"zhaozhou|382", "武汉":"wh|158","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
|
||||
"肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
|
||||
"黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296",
|
||||
"咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517",
|
||||
"仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896",
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
-1
@@ -40,7 +40,6 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
static main(args) {
|
||||
runBatch(new ThirdDataToGatherBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
|
||||
Reference in New Issue
Block a user