Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy
This commit is contained in:
2022-10-24 11:47:53 +08:00
9 changed files with 236 additions and 98 deletions
@@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = ["临夏"]//CcbCommonUtils.citys_all;
def citys = CcbCommonUtils.citys_92;
def rows = []
@@ -75,8 +75,8 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -16,13 +16,15 @@ class ExportOracleData {
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def citys = null
static main(args) {
def task = new ExportOracleData()
task.run()
task.run(args)
}
def run() {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
@@ -131,7 +133,6 @@ class ExportOracleData {
def exportBigdataPoiData() {
def citys = CcbCommonUtils.citys_all
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
@@ -201,7 +202,6 @@ class ExportOracleData {
// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def allRows = []
def citys = CcbCommonUtils.citys_all
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'";
@@ -234,7 +234,6 @@ class ExportOracleData {
}
def exportBigdataDistrictData() {
def citys = CcbCommonUtils.citys_all
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
@@ -276,7 +275,6 @@ class ExportOracleData {
def exportGuidedPrice() {
def citys = CcbCommonUtils.citys_all
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
@@ -315,7 +313,6 @@ class ExportOracleData {
}
def exportCommunityGuidedPrice(def date) {
def citys = CcbCommonUtils.citys_all
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
@@ -355,7 +352,6 @@ class ExportOracleData {
def exportBigdataDeal(def dealDate, def new_dists) {
def allRows = []
def citys = CcbCommonUtils.citys_all
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(new_dists){
@@ -531,7 +527,6 @@ class ExportOracleData {
}
def headers = []
def index = 0
def citys = CcbCommonUtils.citys_all
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
@@ -589,7 +584,6 @@ class ExportOracleData {
def exportBigdataCommunity(){
def citys = CcbCommonUtils.citys_all
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
@@ -649,8 +643,6 @@ class ExportOracleData {
def exportBigdataForsale(publishDate, new_dists){
//["北京","上海"]
def citys = CcbCommonUtils.citys_all;
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
@@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys_252;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
@@ -73,15 +73,15 @@ class TradeStatLog {
}
def run(){
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_92;
def rets = []
rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-06-01"
def date = "2022-07-01"
def endDate = "2022-08-01"
def newDate = "2022-09-01"
def startDate = "2022-07-01"
def date = "2022-08-01"
def endDate = "2022-09-01"
def newDate = "2022-10-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -473,6 +473,7 @@ class CommonDatautils {
if(dbo.mapping_alias){
detailMap.mapping_alias = dbo.mapping_alias;
}
detailMap.estate_type = dbo.estate_type2
}
}catch(Exception e){
@@ -54,7 +54,7 @@ class YouDaDataParser {
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
@@ -1,5 +1,6 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{
}
def rowIndex = 0;
DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018)
DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018)
static main(args) {
runBatch(new DataStandardBatch(), args)
}
@@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
// if(city == "北京"){
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
}
while(flag){
@@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
district = null;
}else{
district = getMatchNameFromMap(data.city, data.region, data.name, data.address)
district = getMatchNameFromMap(data)
if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区”
district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address)
def copyObj = data
copyObj.name = data.name+"小区"
district = getMatchNameFromMap(copyObj)
}
if(!district){
def newName = filterSpacialChar(data.name);//包含特殊符号
if(newName) {
district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address)
def copyObj = data
copyObj.name = newName.replaceAll("_", "")
district = getMatchNameFromMap(copyObj)
if(!district){
def s1 = newName.substring(0, newName.indexOf("_"))
if (s1) {
//特殊符号前的内容匹配
//district = getMatchName(data.city, data.region, s1, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, s1, data.address)
copyObj.name = s1
district = getMatchNameFromMap(copyObj)
if (!district) {
def s2 = newName.replace(s1, "").replaceAll("_", "")
def newName2 = null;
if (s2 && s2.length() >= 3) {
//特殊符号后的内容匹配
// district = getMatchName(data.city, data.region, s2, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, s2, data.address)
copyObj.name = s2
district = getMatchNameFromMap(copyObj)
if (district) {
type = 3;
println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]"
}
} else {
newName2 = newName.replaceAll("_", "")
copyObj.name = newName2
//district = getMatchName(data.city, data.region, newName2, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, newName2, data.address)
district = getMatchNameFromMap(copyObj)
if (district) {
type = 4;
println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]"
@@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{
}
if (!district) {
district = getMatchNameFromMap(data.city, null, newName2, data.address)
copyObj.name = newName2
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if(!district){
district = getMatchNameFromMap(data.city, null, s1, data.address)
copyObj.name = s1
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if (district) {
type = 5;
} else {
if (s2 && s2.length() >= 3) {
district = getMatchNameFromMap(data.city, null, s2, data.address)
copyObj.name = s2
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if (district) {
type = 6;
}
@@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{
}catch (Exception e){
addr = data.address;
}
district = getMatchAddressFromMap(data.city, data.region, addr, data.name)
def copyObj = data
copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
type = 8;
}else{
district = getMatchAddressFromMap(data.city, null, addr, data.name)
copyObj.region = null
district = getMatchAddressFromMap(copyObj)
if(district){
type = 9;
}
@@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{
if(!district){
def districts = getMatchPingYing(data.city, data.region, data.name, addr);
district = checkMatchAddress(districts, addr);
district = checkMatchAddress(districts, addr, data);
if(district){
type = 10;
}else if(districts.size()==1){
district = districts[0];
type = 14
district = getFilerMatchDistrict(districts[0], data);
if(district){
type = 14
}
}
}
if(!district){
district = getOpenApiMatch(data.city, data.region, data.name, addr);
def copyObj = data
copyObj.address = addr
district = getOpenApiMatch(copyObj)
if(!district && addr){
district = getOpenApiMatch(data.city, data.region, addr, data.name);
district = getOpenApiMatch(copyObj)
}
if(district){
if(data.region != district.region){
@@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{
return null;
}
def checkMatchAddress(def districts, def address){
def checkMatchAddress(def districts, def address, def data){
def district = null;
if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){
def tempDist = districts[0];
@@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(district){
return getFilerMatchDistrict(district, data)
}
return district;
}
@@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{
}*/
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
def getMatchNameFromMap(def data){
//查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price
def dbo = findSameNameFromMap(data.city, data.region, data.name);
def addrFlag = null
if(address){
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(data.address){
addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
def distrs = findAliasNameFromMap(data.city, data.region, data.name);
if(distrs && distrs.size()==1){
dbo = distrs[0]
}else if(distrs && distrs.size()>1 && address){
}else if(distrs && distrs.size()>1 && data.address){
for(def d : distrs){
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){
dbo = d;
break;
}
}
}
if(dbo){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
//地址
if(!dbo){
def result = findAddressFromMap(city, region, disName)
def result = findAddressFromMap(data.city, data.region, data.name)
if(result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && address){
}else if(result && result.size()>1 && data.address){
for(def d : result){
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){
dbo = d;
break;
}
}
}
if(dbo){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
}
}
if(dbo){
return getFilerMatchDistrict(dbo, data)
}
return dbo;
}
def getFilerMatchDistrict(dbo, data){
def types = []
if(data.estate_type){
types = getEstateTypes(data.estate_type)
}
if(types.size()>0 && types.size() == 1){
if(types.contains(dbo.estate_type)){
return dbo;
}else{
def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","物业类型不匹配")
THIRD_MATCH_FILTER_COLL.save(data)
}
return null
}
}else{
def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger())
def price = priceMap.get(dbo.estate_type)
if(price && data.avg_price && data.avg_price.toString().isNumber()){
def pre = (data.avg_price-price)*100/price
if(pre>=-20 && pre<=20){
return dbo;
}else{
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","房源价格过滤")
THIRD_PRICE_FILTER_COLL.save(data)
}
return null
}
}
}
return dbo
}
def findHousePrice(def city, def region, def name, def startDate){
def typePriceMap = [:]
try{
String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'"
def houseMap = [:]
ElasticUtils.executeSqlQuery(sql,{ data->
if(data.estate_type){
def key = ""
if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){
key = "普通住宅"
}else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){
key = "商住"
}else if( data.estate_type.contains("别墅")){
key = "别墅"
}else{
key = "其它"
}
def houses = null
if(key){
houses = houseMap.get(key)
if(!houses){
houses = []
}
}
if(data.avg_price){
houses << data.avg_price
houseMap.put(key, houses)
}
}
})
if(houseMap){
houseMap.each{k,v->
if(v && v.size()>=3){
def price = (int)(v.sum()/v.size())
typePriceMap.put(k, price)
}
}
}
}catch(Exception e){
e.printStackTrace()
}
return typePriceMap
}
def getEstateTypes(str){
def types = []
if(str){
if(str.contains("住宅") || str.contains("公寓")){
types << "普通住宅"
}
if(str.contains("商住") || str.contains("商务")){
types << "商住"
}
if(str.contains("别墅")){
types << "别墅"
}
}
return types;
}
/*def getMatchAddress(def city, def region, def address, def name, def cat){
def dbo = CommonDatautils.findSeaweed(city, region, address, cat);
@@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{
}
return dbo;
}*/
def getMatchAddressFromMap(def city, def region, def address, def name){
def getMatchAddressFromMap(def copyObj){
def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
def name = copyObj.name
if(copyObj.address && isStandardRoad(copyObj.address)){
dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address);
}
if(!dbo){
def result = null
if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
if(copyObj.address && isStandardRoad(copyObj.address)){
result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address);
}
if(result && result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
}else if(result && result.size()>1 && copyObj.name){
def score = null;
for(def d : result){
if(d.name == name){
if(d.name == copyObj.name){
dbo = d;
break;
}else{
@@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{
}
}
if(!dbo){
result = findAddrAliasFromMap(city, region, address);
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
@@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(dbo){
return getFilerMatchDistrict(dbo, copyObj)
}
}
return dbo;
}
@@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{
if(arrays[0] && arrays.size()==2){
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ")
if(py1 == py2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}
}
}
@@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name, def address){
if(StringUtils.isNotEmpty(name)){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false)
def getOpenApiMatch(def data){
if(StringUtils.isNotEmpty(data.name)){
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit]
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
return getFilerMatchDistrict(dbo, data)
}
return dbo
}
}
return null;
@@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -948,9 +1088,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays[0] && arrays[0]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
@@ -962,9 +1102,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays[0] && arrays[0]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -990,18 +1130,18 @@ class DataStandardBatch extends BaseBatch{
def alias = v.alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}else if(v && v.mapping_alias){
def mappingAlias = v.mapping_alias
if(mappingAlias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1016,18 +1156,18 @@ class DataStandardBatch extends BaseBatch{
def alias = v.alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}else if(v && v.mapping_alias){
def mappingAlias = v.mapping_alias
if(mappingAlias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1052,9 +1192,9 @@ class DataStandardBatch extends BaseBatch{
def alias = v.addr_alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1068,9 +1208,9 @@ class DataStandardBatch extends BaseBatch{
def alias = v.addr_alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1093,9 +1233,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays && arrays.size()==2 && arrays[1]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1106,9 +1246,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays && arrays.size()==2 && arrays[1]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -24,8 +24,8 @@ class SynchDistrictPrice extends BaseBatch{
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
// "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
def esSources = ["中原地产":"中原地产"]
static main(args) {
@@ -57,7 +57,7 @@ class SynchDistrictPrice extends BaseBatch{
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
def s = d.city +"#"+d.region+"#"+d.name
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
if(!names.contains(s)){
names << s
def data = new BasicDBObject()
@@ -67,6 +67,7 @@ class SynchDistrictPrice extends BaseBatch{
data.put("address", d.address)
data.put("avg_price", d.avg_price)
data.put("source", mgSource)
data.put("estate_type",d.property_type)
data.put("s_date", SHORT_DATE)
data.put("url", d.url)
data.put("c_date", new Date())
@@ -144,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
}else{
for(def r : rets){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source)
if(obj){
updateIndex++
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
@@ -152,6 +153,7 @@ class SynchDistrictPrice extends BaseBatch{
def udata = new BasicDBObject()
udata.put("avg_price", r.avg_price)
udata.put("estate_type", r.estate_type)
udata.put("m_date",new Date())
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
@@ -181,13 +183,16 @@ class SynchDistrictPrice extends BaseBatch{
}
def findIsExists(def city, def region, def name, def sDate, def source){
def findIsExists(def city, def region, def name, estate_type, def sDate, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", sDate)
q.put("city", city)
q.put("region", region)
q.put("name", name)
if(estate_type){
q.put("estate_type",estate_type)
}
def obj = thirdPartyColl.findOne(q)
return obj
}