建行数据
This commit is contained in:
+1
-1
@@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["临夏"]//CcbCommonUtils.citys_all;
|
||||
def citys = CcbCommonUtils.citys_92;
|
||||
|
||||
def rows = []
|
||||
|
||||
|
||||
+2
-2
@@ -74,8 +74,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+1
@@ -473,6 +473,7 @@ class CommonDatautils {
|
||||
if(dbo.mapping_alias){
|
||||
detailMap.mapping_alias = dbo.mapping_alias;
|
||||
}
|
||||
detailMap.estate_type = dbo.estate_type2
|
||||
|
||||
}
|
||||
}catch(Exception e){
|
||||
|
||||
+207
-69
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.thirdPriceCrawl
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
def rowIndex = 0;
|
||||
|
||||
DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018)
|
||||
DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018)
|
||||
static main(args) {
|
||||
runBatch(new DataStandardBatch(), args)
|
||||
}
|
||||
@@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{
|
||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||
|
||||
cityMap.each{city,alias->
|
||||
// if(city == "北京"){
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{
|
||||
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
|
||||
district = null;
|
||||
}else{
|
||||
district = getMatchNameFromMap(data.city, data.region, data.name, data.address)
|
||||
district = getMatchNameFromMap(data)
|
||||
if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区”
|
||||
district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address)
|
||||
def copyObj = data
|
||||
copyObj.name = data.name+"小区"
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
}
|
||||
|
||||
if(!district){
|
||||
def newName = filterSpacialChar(data.name);//包含特殊符号
|
||||
|
||||
if(newName) {
|
||||
district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address)
|
||||
def copyObj = data
|
||||
copyObj.name = newName.replaceAll("_", "")
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
|
||||
if(!district){
|
||||
def s1 = newName.substring(0, newName.indexOf("_"))
|
||||
if (s1) {
|
||||
//特殊符号前的内容匹配
|
||||
//district = getMatchName(data.city, data.region, s1, data.address,"district")
|
||||
district = getMatchNameFromMap(data.city, data.region, s1, data.address)
|
||||
copyObj.name = s1
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
if (!district) {
|
||||
|
||||
def s2 = newName.replace(s1, "").replaceAll("_", "")
|
||||
def newName2 = null;
|
||||
if (s2 && s2.length() >= 3) {
|
||||
//特殊符号后的内容匹配
|
||||
// district = getMatchName(data.city, data.region, s2, data.address,"district")
|
||||
district = getMatchNameFromMap(data.city, data.region, s2, data.address)
|
||||
copyObj.name = s2
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
if (district) {
|
||||
type = 3;
|
||||
println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]"
|
||||
}
|
||||
} else {
|
||||
newName2 = newName.replaceAll("_", "")
|
||||
copyObj.name = newName2
|
||||
//district = getMatchName(data.city, data.region, newName2, data.address,"district")
|
||||
district = getMatchNameFromMap(data.city, data.region, newName2, data.address)
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
if (district) {
|
||||
type = 4;
|
||||
println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]"
|
||||
@@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
if (!district) {
|
||||
|
||||
district = getMatchNameFromMap(data.city, null, newName2, data.address)
|
||||
copyObj.name = newName2
|
||||
copyObj.region = null
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
if(!district){
|
||||
district = getMatchNameFromMap(data.city, null, s1, data.address)
|
||||
copyObj.name = s1
|
||||
copyObj.region = null
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
if (district) {
|
||||
type = 5;
|
||||
} else {
|
||||
if (s2 && s2.length() >= 3) {
|
||||
district = getMatchNameFromMap(data.city, null, s2, data.address)
|
||||
copyObj.name = s2
|
||||
copyObj.region = null
|
||||
district = getMatchNameFromMap(copyObj)
|
||||
if (district) {
|
||||
type = 6;
|
||||
}
|
||||
@@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{
|
||||
}catch (Exception e){
|
||||
addr = data.address;
|
||||
}
|
||||
district = getMatchAddressFromMap(data.city, data.region, addr, data.name)
|
||||
def copyObj = data
|
||||
copyObj.address = addr
|
||||
district = getMatchAddressFromMap(copyObj)
|
||||
if(district){
|
||||
type = 8;
|
||||
}else{
|
||||
district = getMatchAddressFromMap(data.city, null, addr, data.name)
|
||||
copyObj.region = null
|
||||
district = getMatchAddressFromMap(copyObj)
|
||||
if(district){
|
||||
type = 9;
|
||||
}
|
||||
@@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
if(!district){
|
||||
def districts = getMatchPingYing(data.city, data.region, data.name, addr);
|
||||
district = checkMatchAddress(districts, addr);
|
||||
district = checkMatchAddress(districts, addr, data);
|
||||
if(district){
|
||||
type = 10;
|
||||
}else if(districts.size()==1){
|
||||
district = districts[0];
|
||||
type = 14
|
||||
district = getFilerMatchDistrict(districts[0], data.estate_type, avg_price);
|
||||
if(district){
|
||||
type = 14
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!district){
|
||||
district = getOpenApiMatch(data.city, data.region, data.name, addr);
|
||||
def copyObj = data
|
||||
copyObj.address = addr
|
||||
district = getOpenApiMatch(copyObj)
|
||||
if(!district && addr){
|
||||
district = getOpenApiMatch(data.city, data.region, addr, data.name);
|
||||
district = getOpenApiMatch(copyObj)
|
||||
}
|
||||
if(district){
|
||||
if(data.region != district.region){
|
||||
@@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
return null;
|
||||
}
|
||||
|
||||
def checkMatchAddress(def districts, def address){
|
||||
def checkMatchAddress(def districts, def address, def data){
|
||||
def district = null;
|
||||
if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){
|
||||
def tempDist = districts[0];
|
||||
@@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(district){
|
||||
return getFilerMatchDistrict(district, data.estate_type, data.avg_price)
|
||||
}
|
||||
return district;
|
||||
}
|
||||
|
||||
@@ -584,54 +611,152 @@ class DataStandardBatch extends BaseBatch{
|
||||
}*/
|
||||
|
||||
|
||||
def getMatchNameFromMap(def city, def region, def disName, String address){
|
||||
//查询房价网是否存在该小区
|
||||
def dbo = findSameNameFromMap(city, region, disName);
|
||||
def getMatchNameFromMap(def data){
|
||||
//查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price
|
||||
def dbo = findSameNameFromMap(data.city, data.region, data.name);
|
||||
def addrFlag = null
|
||||
if(address){
|
||||
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(data.address){
|
||||
addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
dbo = addressCheck(dbo, data.region, data.address);
|
||||
}
|
||||
}
|
||||
if(!dbo){
|
||||
//别名
|
||||
def distrs = findAliasNameFromMap(city, region, disName);
|
||||
def distrs = findAliasNameFromMap(data.city, data.region, data.name);
|
||||
if(distrs && distrs.size()==1){
|
||||
dbo = distrs[0]
|
||||
}else if(distrs && distrs.size()>1 && address){
|
||||
}else if(distrs && distrs.size()>1 && data.address){
|
||||
for(def d : distrs){
|
||||
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
|
||||
if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){
|
||||
dbo = d;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(dbo){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
dbo = addressCheck(dbo, data.region, data.address);
|
||||
}
|
||||
|
||||
//地址
|
||||
if(!dbo){
|
||||
def result = findAddressFromMap(city, region, disName)
|
||||
def result = findAddressFromMap(data.city, data.region, data.name)
|
||||
if(result.size()==1){
|
||||
dbo = result[0];
|
||||
}else if(result && result.size()>1 && address){
|
||||
}else if(result && result.size()>1 && data.address){
|
||||
for(def d : result){
|
||||
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
|
||||
if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){
|
||||
dbo = d;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(dbo){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
dbo = addressCheck(dbo, data.region, data.address);
|
||||
}
|
||||
}
|
||||
}
|
||||
if(dbo){
|
||||
return getFilerMatchDistrict(dbo, data)
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
def getFilerMatchDistrict(dbo, data){
|
||||
def types = []
|
||||
if(data.estate_type){
|
||||
types = getEstateTypes(data.estate_type)
|
||||
}
|
||||
if(types.size()>0 && types.size() == 1){
|
||||
if(types.contains(dbo.estate_type)){
|
||||
return dbo;
|
||||
}else{
|
||||
def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id)
|
||||
if(!ret){
|
||||
data.put("msg","物业类型不匹配")
|
||||
THIRD_MATCH_FILTER_COLL.save(data)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}else{
|
||||
def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger())
|
||||
def price = priceMap.get(dbo.estate_type)
|
||||
if(price){
|
||||
def pre = (data.avg_price-price)*100/price
|
||||
if(pre>=-20 && pre<=20){
|
||||
return dbo;
|
||||
}else{
|
||||
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
|
||||
if(!ret){
|
||||
data.put("msg","房源价格过滤")
|
||||
THIRD_PRICE_FILTER_COLL.save(data)
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
return dbo
|
||||
}
|
||||
|
||||
|
||||
def findHousePrice(def city, def region, def name, def startDate){
|
||||
def typePriceMap = [:]
|
||||
try{
|
||||
String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'"
|
||||
def houseMap = [:]
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
if(data.estate_type){
|
||||
def key = ""
|
||||
if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){
|
||||
key = "普通住宅"
|
||||
}else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){
|
||||
key = "商住"
|
||||
}else if( data.estate_type.contains("别墅")){
|
||||
key = "别墅"
|
||||
}
|
||||
def houses = null
|
||||
if(key){
|
||||
houses = houseMap.get(key)
|
||||
if(!houses){
|
||||
houses = []
|
||||
}
|
||||
}
|
||||
if(data.avg_price){
|
||||
houses << data.avg_price
|
||||
houseMap.put(key, houses)
|
||||
}
|
||||
}
|
||||
})
|
||||
if(houseMap){
|
||||
houseMap.each{k,v->
|
||||
if(v && v.size()>=3){
|
||||
def price = (int)(v.sum()/v.size())
|
||||
typePriceMap.put(k, price)
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return typePriceMap
|
||||
}
|
||||
|
||||
def getEstateTypes(str){
|
||||
def types = []
|
||||
if(str){
|
||||
if(str.contains("住宅") || str.contains("公寓")){
|
||||
types << "普通住宅"
|
||||
}
|
||||
if(str.contains("商住") || str.contains("商务")){
|
||||
types << "商住"
|
||||
}
|
||||
if(str.contains("别墅")){
|
||||
types << "别墅"
|
||||
}
|
||||
}
|
||||
return types;
|
||||
}
|
||||
/*def getMatchAddress(def city, def region, def address, def name, def cat){
|
||||
def dbo = CommonDatautils.findSeaweed(city, region, address, cat);
|
||||
|
||||
@@ -664,22 +789,23 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
return dbo;
|
||||
}*/
|
||||
def getMatchAddressFromMap(def city, def region, def address, def name){
|
||||
def getMatchAddressFromMap(def copyObj){
|
||||
def dbo = null
|
||||
if(address && isStandardRoad(address)){
|
||||
dbo = findSameNameFromMap(city, region, address);
|
||||
def name = copyObj.name
|
||||
if(copyObj.address && isStandardRoad(copyObj.address)){
|
||||
dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||
}
|
||||
if(!dbo){
|
||||
def result = null
|
||||
if(address && isStandardRoad(address)){
|
||||
result = findAddressFromMap(city, region, address);
|
||||
if(copyObj.address && isStandardRoad(copyObj.address)){
|
||||
result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||
}
|
||||
if(result && result.size()==1){
|
||||
dbo = result[0];
|
||||
}else if(result && result.size()>1 && name){
|
||||
}else if(result && result.size()>1 && copyObj.name){
|
||||
def score = null;
|
||||
for(def d : result){
|
||||
if(d.name == name){
|
||||
if(d.name == copyObj.name){
|
||||
dbo = d;
|
||||
break;
|
||||
}else{
|
||||
@@ -706,7 +832,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
if(!dbo){
|
||||
result = findAddrAliasFromMap(city, region, address);
|
||||
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||
if(result.size()==1){
|
||||
dbo = result[0];
|
||||
}else if(result && result.size()>1 && name){
|
||||
@@ -718,6 +844,11 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
return getFilerMatchDistrict(dbo, copyObj)
|
||||
}
|
||||
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
@@ -756,7 +887,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
if(arrays[0] && arrays.size()==2){
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ")
|
||||
if(py1 == py2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -768,11 +899,15 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def getOpenApiMatch(def city, def region, def name, def address){
|
||||
if(StringUtils.isNotEmpty(name)){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false)
|
||||
def getOpenApiMatch(def data){
|
||||
if(StringUtils.isNotEmpty(data.name)){
|
||||
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit]
|
||||
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
|
||||
if(dbo){
|
||||
return getFilerMatchDistrict(dbo, data)
|
||||
}
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
return null;
|
||||
@@ -783,6 +918,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("city","北京")
|
||||
qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
def ret = []
|
||||
try{
|
||||
@@ -948,9 +1086,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def arrays = k.split("###")
|
||||
if(arrays[0] && arrays[0]==keyword){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:region, name:arrays[0]]
|
||||
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
|
||||
}
|
||||
@@ -962,9 +1100,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def arrays = k.split("###")
|
||||
if(arrays[0] && arrays[0]==keyword){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:region, name:arrays[0]]
|
||||
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -990,18 +1128,18 @@ class DataStandardBatch extends BaseBatch{
|
||||
def alias = v.alias;
|
||||
if(alias.contains(keyword)){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:region, name:arrays[0]]
|
||||
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}else if(v && v.mapping_alias){
|
||||
def mappingAlias = v.mapping_alias
|
||||
if(mappingAlias.contains(keyword)){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:region, name:arrays[0]]
|
||||
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1016,18 +1154,18 @@ class DataStandardBatch extends BaseBatch{
|
||||
def alias = v.alias;
|
||||
if(alias.contains(keyword)){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:r, name:arrays[0]]
|
||||
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}else if(v && v.mapping_alias){
|
||||
def mappingAlias = v.mapping_alias
|
||||
if(mappingAlias.contains(keyword)){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:r, name:arrays[0]]
|
||||
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1052,9 +1190,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def alias = v.addr_alias;
|
||||
if(alias.contains(keyword)){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:region, name:arrays[0]]
|
||||
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1068,9 +1206,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def alias = v.addr_alias;
|
||||
if(alias.contains(keyword)){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:r, name:arrays[0]]
|
||||
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1093,9 +1231,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def arrays = k.split("###")
|
||||
if(arrays && arrays.size()==2 && arrays[1]==keyword){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:region, name:arrays[0]]
|
||||
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1106,9 +1244,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def arrays = k.split("###")
|
||||
if(arrays && arrays.size()==2 && arrays[1]==keyword){
|
||||
if(arrays.size()>=2){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||
}else{
|
||||
districts << [city:city, region:r, name:arrays[0]]
|
||||
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+10
-5
@@ -24,8 +24,8 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
|
||||
// "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
|
||||
def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
|
||||
def esSources = ["中原地产":"中原地产"]
|
||||
|
||||
|
||||
static main(args) {
|
||||
@@ -57,7 +57,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
def data = new BasicDBObject()
|
||||
@@ -67,6 +67,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
data.put("address", d.address)
|
||||
data.put("avg_price", d.avg_price)
|
||||
data.put("source", mgSource)
|
||||
data.put("estate_type",d.property_type)
|
||||
data.put("s_date", SHORT_DATE)
|
||||
data.put("url", d.url)
|
||||
data.put("c_date", new Date())
|
||||
@@ -144,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
|
||||
}else{
|
||||
for(def r : rets){
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source)
|
||||
if(obj){
|
||||
updateIndex++
|
||||
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
@@ -152,6 +153,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("avg_price", r.avg_price)
|
||||
udata.put("estate_type", r.estate_type)
|
||||
udata.put("m_date",new Date())
|
||||
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
|
||||
|
||||
@@ -181,13 +183,16 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def findIsExists(def city, def region, def name, def sDate, def source){
|
||||
def findIsExists(def city, def region, def name, estate_type, def sDate, def source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", source)
|
||||
q.put("s_date", sDate)
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
if(estate_type){
|
||||
q.put("estate_type",estate_type)
|
||||
}
|
||||
def obj = thirdPartyColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user