Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
# Conflicts: # projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy
This commit is contained in:
+1
-1
@@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def citys = ["临夏"]//CcbCommonUtils.citys_all;
|
def citys = CcbCommonUtils.citys_92;
|
||||||
|
|
||||||
def rows = []
|
def rows = []
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -75,8 +75,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
|
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
|
||||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||||
//测试通过匹配接口配房源效果
|
//测试通过匹配接口配房源效果
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||||
|
|||||||
+4
-12
@@ -16,13 +16,15 @@ class ExportOracleData {
|
|||||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
def citys = null
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new ExportOracleData()
|
def task = new ExportOracleData()
|
||||||
task.run()
|
task.run(args)
|
||||||
}
|
}
|
||||||
|
|
||||||
def run() {
|
def run(args) {
|
||||||
|
citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||||
@@ -131,7 +133,6 @@ class ExportOracleData {
|
|||||||
|
|
||||||
|
|
||||||
def exportBigdataPoiData() {
|
def exportBigdataPoiData() {
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
def allRows = []
|
def allRows = []
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
@@ -201,7 +202,6 @@ class ExportOracleData {
|
|||||||
// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
def allRows = []
|
def allRows = []
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'";
|
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'";
|
||||||
@@ -234,7 +234,6 @@ class ExportOracleData {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def exportBigdataDistrictData() {
|
def exportBigdataDistrictData() {
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
def allRows = []
|
def allRows = []
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
@@ -276,7 +275,6 @@ class ExportOracleData {
|
|||||||
|
|
||||||
|
|
||||||
def exportGuidedPrice() {
|
def exportGuidedPrice() {
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
def allRows = []
|
def allRows = []
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
@@ -315,7 +313,6 @@ class ExportOracleData {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def exportCommunityGuidedPrice(def date) {
|
def exportCommunityGuidedPrice(def date) {
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
def allRows = []
|
def allRows = []
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
@@ -355,7 +352,6 @@ class ExportOracleData {
|
|||||||
def exportBigdataDeal(def dealDate, def new_dists) {
|
def exportBigdataDeal(def dealDate, def new_dists) {
|
||||||
|
|
||||||
def allRows = []
|
def allRows = []
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
if(new_dists){
|
if(new_dists){
|
||||||
@@ -531,7 +527,6 @@ class ExportOracleData {
|
|||||||
}
|
}
|
||||||
def headers = []
|
def headers = []
|
||||||
def index = 0
|
def index = 0
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
def allRows = []
|
def allRows = []
|
||||||
@@ -589,7 +584,6 @@ class ExportOracleData {
|
|||||||
|
|
||||||
def exportBigdataCommunity(){
|
def exportBigdataCommunity(){
|
||||||
|
|
||||||
def citys = CcbCommonUtils.citys_all
|
|
||||||
def allRows = []
|
def allRows = []
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeMap.get(city)
|
def cityCode = cityCodeMap.get(city)
|
||||||
@@ -649,8 +643,6 @@ class ExportOracleData {
|
|||||||
|
|
||||||
def exportBigdataForsale(publishDate, new_dists){
|
def exportBigdataForsale(publishDate, new_dists){
|
||||||
//["北京","上海"]
|
//["北京","上海"]
|
||||||
def citys = CcbCommonUtils.citys_all;
|
|
||||||
|
|
||||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||||
if(CommonDatautils.isWindow()){
|
if(CommonDatautils.isWindow()){
|
||||||
|
|||||||
+1
-1
@@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def citys = CcbCommonUtils.citys_252;
|
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
def rows = []
|
def rows = []
|
||||||
def index = 0
|
def index = 0
|
||||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||||
|
|||||||
+7
-7
@@ -73,15 +73,15 @@ class TradeStatLog {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def citys = CcbCommonUtils.citys;
|
def citys = CcbCommonUtils.citys_92;
|
||||||
def rets = []
|
def rets = []
|
||||||
rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||||
//def startDate = "2021-05-01"
|
//def startDate = "2021-05-01"
|
||||||
def startDate = "2022-06-01"
|
def startDate = "2022-07-01"
|
||||||
def date = "2022-07-01"
|
def date = "2022-08-01"
|
||||||
def endDate = "2022-08-01"
|
def endDate = "2022-09-01"
|
||||||
def newDate = "2022-09-01"
|
def newDate = "2022-10-01"
|
||||||
|
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||||
|
|||||||
+1
@@ -473,6 +473,7 @@ class CommonDatautils {
|
|||||||
if(dbo.mapping_alias){
|
if(dbo.mapping_alias){
|
||||||
detailMap.mapping_alias = dbo.mapping_alias;
|
detailMap.mapping_alias = dbo.mapping_alias;
|
||||||
}
|
}
|
||||||
|
detailMap.estate_type = dbo.estate_type2
|
||||||
|
|
||||||
}
|
}
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
|
|||||||
+1
-1
@@ -54,7 +54,7 @@ class YouDaDataParser {
|
|||||||
// task.newHouseparse();//友达一手房存入mongo
|
// task.newHouseparse();//友达一手房存入mongo
|
||||||
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
|
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
|
||||||
// task.newhouseToSaveEs();//新房保存到es
|
// task.newhouseToSaveEs();//新房保存到es
|
||||||
task.esfhouseToSaveEs();//二手房保存到es
|
// task.esfhouseToSaveEs();//二手房保存到es
|
||||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+209
-69
@@ -1,5 +1,6 @@
|
|||||||
package com.fangjia.script.thirdPriceCrawl
|
package com.fangjia.script.thirdPriceCrawl
|
||||||
|
|
||||||
|
import com.fangjia.core.util.DateUtil
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
@@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils
|
|||||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||||
import com.fangjia.script.util.DataUtil
|
import com.fangjia.script.util.DataUtil
|
||||||
import com.fangjia.script.util.DbUtils
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.fangjia.script.util.OpenApiClient
|
import com.fangjia.script.util.OpenApiClient
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
@@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
}
|
}
|
||||||
def rowIndex = 0;
|
def rowIndex = 0;
|
||||||
|
|
||||||
|
DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018)
|
||||||
|
DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018)
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new DataStandardBatch(), args)
|
runBatch(new DataStandardBatch(), args)
|
||||||
}
|
}
|
||||||
@@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||||
|
|
||||||
cityMap.each{city,alias->
|
cityMap.each{city,alias->
|
||||||
|
// if(city == "北京"){
|
||||||
println "开始加载【${city}】"
|
println "开始加载【${city}】"
|
||||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||||
districtMap.putAll(detailMap)
|
districtMap.putAll(detailMap)
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
while(flag){
|
while(flag){
|
||||||
@@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
|
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
|
||||||
district = null;
|
district = null;
|
||||||
}else{
|
}else{
|
||||||
district = getMatchNameFromMap(data.city, data.region, data.name, data.address)
|
district = getMatchNameFromMap(data)
|
||||||
if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区”
|
if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区”
|
||||||
district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address)
|
def copyObj = data
|
||||||
|
copyObj.name = data.name+"小区"
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!district){
|
if(!district){
|
||||||
def newName = filterSpacialChar(data.name);//包含特殊符号
|
def newName = filterSpacialChar(data.name);//包含特殊符号
|
||||||
|
|
||||||
if(newName) {
|
if(newName) {
|
||||||
district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address)
|
def copyObj = data
|
||||||
|
copyObj.name = newName.replaceAll("_", "")
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
|
|
||||||
if(!district){
|
if(!district){
|
||||||
def s1 = newName.substring(0, newName.indexOf("_"))
|
def s1 = newName.substring(0, newName.indexOf("_"))
|
||||||
if (s1) {
|
if (s1) {
|
||||||
//特殊符号前的内容匹配
|
//特殊符号前的内容匹配
|
||||||
//district = getMatchName(data.city, data.region, s1, data.address,"district")
|
//district = getMatchName(data.city, data.region, s1, data.address,"district")
|
||||||
district = getMatchNameFromMap(data.city, data.region, s1, data.address)
|
copyObj.name = s1
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
if (!district) {
|
if (!district) {
|
||||||
|
|
||||||
def s2 = newName.replace(s1, "").replaceAll("_", "")
|
def s2 = newName.replace(s1, "").replaceAll("_", "")
|
||||||
def newName2 = null;
|
def newName2 = null;
|
||||||
if (s2 && s2.length() >= 3) {
|
if (s2 && s2.length() >= 3) {
|
||||||
//特殊符号后的内容匹配
|
//特殊符号后的内容匹配
|
||||||
// district = getMatchName(data.city, data.region, s2, data.address,"district")
|
// district = getMatchName(data.city, data.region, s2, data.address,"district")
|
||||||
district = getMatchNameFromMap(data.city, data.region, s2, data.address)
|
copyObj.name = s2
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
if (district) {
|
if (district) {
|
||||||
type = 3;
|
type = 3;
|
||||||
println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]"
|
println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]"
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
newName2 = newName.replaceAll("_", "")
|
newName2 = newName.replaceAll("_", "")
|
||||||
|
copyObj.name = newName2
|
||||||
//district = getMatchName(data.city, data.region, newName2, data.address,"district")
|
//district = getMatchName(data.city, data.region, newName2, data.address,"district")
|
||||||
district = getMatchNameFromMap(data.city, data.region, newName2, data.address)
|
district = getMatchNameFromMap(copyObj)
|
||||||
if (district) {
|
if (district) {
|
||||||
type = 4;
|
type = 4;
|
||||||
println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]"
|
println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]"
|
||||||
@@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!district) {
|
if (!district) {
|
||||||
|
copyObj.name = newName2
|
||||||
district = getMatchNameFromMap(data.city, null, newName2, data.address)
|
copyObj.region = null
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
if(!district){
|
if(!district){
|
||||||
district = getMatchNameFromMap(data.city, null, s1, data.address)
|
copyObj.name = s1
|
||||||
|
copyObj.region = null
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
if (district) {
|
if (district) {
|
||||||
type = 5;
|
type = 5;
|
||||||
} else {
|
} else {
|
||||||
if (s2 && s2.length() >= 3) {
|
if (s2 && s2.length() >= 3) {
|
||||||
district = getMatchNameFromMap(data.city, null, s2, data.address)
|
copyObj.name = s2
|
||||||
|
copyObj.region = null
|
||||||
|
district = getMatchNameFromMap(copyObj)
|
||||||
if (district) {
|
if (district) {
|
||||||
type = 6;
|
type = 6;
|
||||||
}
|
}
|
||||||
@@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}catch (Exception e){
|
}catch (Exception e){
|
||||||
addr = data.address;
|
addr = data.address;
|
||||||
}
|
}
|
||||||
district = getMatchAddressFromMap(data.city, data.region, addr, data.name)
|
def copyObj = data
|
||||||
|
copyObj.address = addr
|
||||||
|
district = getMatchAddressFromMap(copyObj)
|
||||||
if(district){
|
if(district){
|
||||||
type = 8;
|
type = 8;
|
||||||
}else{
|
}else{
|
||||||
district = getMatchAddressFromMap(data.city, null, addr, data.name)
|
copyObj.region = null
|
||||||
|
district = getMatchAddressFromMap(copyObj)
|
||||||
if(district){
|
if(district){
|
||||||
type = 9;
|
type = 9;
|
||||||
}
|
}
|
||||||
@@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
if(!district){
|
if(!district){
|
||||||
def districts = getMatchPingYing(data.city, data.region, data.name, addr);
|
def districts = getMatchPingYing(data.city, data.region, data.name, addr);
|
||||||
district = checkMatchAddress(districts, addr);
|
district = checkMatchAddress(districts, addr, data);
|
||||||
if(district){
|
if(district){
|
||||||
type = 10;
|
type = 10;
|
||||||
}else if(districts.size()==1){
|
}else if(districts.size()==1){
|
||||||
district = districts[0];
|
district = getFilerMatchDistrict(districts[0], data);
|
||||||
type = 14
|
if(district){
|
||||||
|
type = 14
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!district){
|
if(!district){
|
||||||
district = getOpenApiMatch(data.city, data.region, data.name, addr);
|
def copyObj = data
|
||||||
|
copyObj.address = addr
|
||||||
|
district = getOpenApiMatch(copyObj)
|
||||||
if(!district && addr){
|
if(!district && addr){
|
||||||
district = getOpenApiMatch(data.city, data.region, addr, data.name);
|
district = getOpenApiMatch(copyObj)
|
||||||
}
|
}
|
||||||
if(district){
|
if(district){
|
||||||
if(data.region != district.region){
|
if(data.region != district.region){
|
||||||
@@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
def checkMatchAddress(def districts, def address){
|
def checkMatchAddress(def districts, def address, def data){
|
||||||
def district = null;
|
def district = null;
|
||||||
if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){
|
if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){
|
||||||
def tempDist = districts[0];
|
def tempDist = districts[0];
|
||||||
@@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(district){
|
||||||
|
return getFilerMatchDistrict(district, data)
|
||||||
|
}
|
||||||
return district;
|
return district;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}*/
|
}*/
|
||||||
|
|
||||||
|
|
||||||
def getMatchNameFromMap(def city, def region, def disName, String address){
|
def getMatchNameFromMap(def data){
|
||||||
//查询房价网是否存在该小区
|
//查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price
|
||||||
def dbo = findSameNameFromMap(city, region, disName);
|
def dbo = findSameNameFromMap(data.city, data.region, data.name);
|
||||||
def addrFlag = null
|
def addrFlag = null
|
||||||
if(address){
|
if(data.address){
|
||||||
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||||
if(addrFlag){
|
if(addrFlag){
|
||||||
dbo = addressCheck(dbo, region, address);
|
dbo = addressCheck(dbo, data.region, data.address);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(!dbo){
|
if(!dbo){
|
||||||
//别名
|
//别名
|
||||||
def distrs = findAliasNameFromMap(city, region, disName);
|
def distrs = findAliasNameFromMap(data.city, data.region, data.name);
|
||||||
if(distrs && distrs.size()==1){
|
if(distrs && distrs.size()==1){
|
||||||
dbo = distrs[0]
|
dbo = distrs[0]
|
||||||
}else if(distrs && distrs.size()>1 && address){
|
}else if(distrs && distrs.size()>1 && data.address){
|
||||||
for(def d : distrs){
|
for(def d : distrs){
|
||||||
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
|
if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){
|
||||||
dbo = d;
|
dbo = d;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(dbo){
|
if(dbo){
|
||||||
dbo = addressCheck(dbo, region, address);
|
dbo = addressCheck(dbo, data.region, data.address);
|
||||||
}
|
}
|
||||||
|
|
||||||
//地址
|
//地址
|
||||||
if(!dbo){
|
if(!dbo){
|
||||||
def result = findAddressFromMap(city, region, disName)
|
def result = findAddressFromMap(data.city, data.region, data.name)
|
||||||
if(result.size()==1){
|
if(result.size()==1){
|
||||||
dbo = result[0];
|
dbo = result[0];
|
||||||
}else if(result && result.size()>1 && address){
|
}else if(result && result.size()>1 && data.address){
|
||||||
for(def d : result){
|
for(def d : result){
|
||||||
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
|
if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){
|
||||||
dbo = d;
|
dbo = d;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(dbo){
|
if(dbo){
|
||||||
dbo = addressCheck(dbo, region, address);
|
dbo = addressCheck(dbo, data.region, data.address);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if(dbo){
|
||||||
|
return getFilerMatchDistrict(dbo, data)
|
||||||
|
}
|
||||||
return dbo;
|
return dbo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getFilerMatchDistrict(dbo, data){
|
||||||
|
def types = []
|
||||||
|
if(data.estate_type){
|
||||||
|
types = getEstateTypes(data.estate_type)
|
||||||
|
}
|
||||||
|
if(types.size()>0 && types.size() == 1){
|
||||||
|
if(types.contains(dbo.estate_type)){
|
||||||
|
return dbo;
|
||||||
|
}else{
|
||||||
|
def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id)
|
||||||
|
if(!ret){
|
||||||
|
data.put("msg","物业类型不匹配")
|
||||||
|
THIRD_MATCH_FILTER_COLL.save(data)
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger())
|
||||||
|
def price = priceMap.get(dbo.estate_type)
|
||||||
|
if(price && data.avg_price && data.avg_price.toString().isNumber()){
|
||||||
|
def pre = (data.avg_price-price)*100/price
|
||||||
|
if(pre>=-20 && pre<=20){
|
||||||
|
return dbo;
|
||||||
|
}else{
|
||||||
|
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
|
||||||
|
if(!ret){
|
||||||
|
data.put("msg","房源价格过滤")
|
||||||
|
THIRD_PRICE_FILTER_COLL.save(data)
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dbo
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findHousePrice(def city, def region, def name, def startDate){
|
||||||
|
def typePriceMap = [:]
|
||||||
|
try{
|
||||||
|
String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'"
|
||||||
|
def houseMap = [:]
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||||
|
if(data.estate_type){
|
||||||
|
def key = ""
|
||||||
|
if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){
|
||||||
|
key = "普通住宅"
|
||||||
|
}else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){
|
||||||
|
key = "商住"
|
||||||
|
}else if( data.estate_type.contains("别墅")){
|
||||||
|
key = "别墅"
|
||||||
|
}else{
|
||||||
|
key = "其它"
|
||||||
|
}
|
||||||
|
def houses = null
|
||||||
|
if(key){
|
||||||
|
houses = houseMap.get(key)
|
||||||
|
if(!houses){
|
||||||
|
houses = []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(data.avg_price){
|
||||||
|
houses << data.avg_price
|
||||||
|
houseMap.put(key, houses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if(houseMap){
|
||||||
|
houseMap.each{k,v->
|
||||||
|
if(v && v.size()>=3){
|
||||||
|
def price = (int)(v.sum()/v.size())
|
||||||
|
typePriceMap.put(k, price)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
e.printStackTrace()
|
||||||
|
}
|
||||||
|
|
||||||
|
return typePriceMap
|
||||||
|
}
|
||||||
|
|
||||||
|
def getEstateTypes(str){
|
||||||
|
def types = []
|
||||||
|
if(str){
|
||||||
|
if(str.contains("住宅") || str.contains("公寓")){
|
||||||
|
types << "普通住宅"
|
||||||
|
}
|
||||||
|
if(str.contains("商住") || str.contains("商务")){
|
||||||
|
types << "商住"
|
||||||
|
}
|
||||||
|
if(str.contains("别墅")){
|
||||||
|
types << "别墅"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return types;
|
||||||
|
}
|
||||||
/*def getMatchAddress(def city, def region, def address, def name, def cat){
|
/*def getMatchAddress(def city, def region, def address, def name, def cat){
|
||||||
def dbo = CommonDatautils.findSeaweed(city, region, address, cat);
|
def dbo = CommonDatautils.findSeaweed(city, region, address, cat);
|
||||||
|
|
||||||
@@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
return dbo;
|
return dbo;
|
||||||
}*/
|
}*/
|
||||||
def getMatchAddressFromMap(def city, def region, def address, def name){
|
def getMatchAddressFromMap(def copyObj){
|
||||||
def dbo = null
|
def dbo = null
|
||||||
if(address && isStandardRoad(address)){
|
def name = copyObj.name
|
||||||
dbo = findSameNameFromMap(city, region, address);
|
if(copyObj.address && isStandardRoad(copyObj.address)){
|
||||||
|
dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||||
}
|
}
|
||||||
if(!dbo){
|
if(!dbo){
|
||||||
def result = null
|
def result = null
|
||||||
if(address && isStandardRoad(address)){
|
if(copyObj.address && isStandardRoad(copyObj.address)){
|
||||||
result = findAddressFromMap(city, region, address);
|
result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||||
}
|
}
|
||||||
if(result && result.size()==1){
|
if(result && result.size()==1){
|
||||||
dbo = result[0];
|
dbo = result[0];
|
||||||
}else if(result && result.size()>1 && name){
|
}else if(result && result.size()>1 && copyObj.name){
|
||||||
def score = null;
|
def score = null;
|
||||||
for(def d : result){
|
for(def d : result){
|
||||||
if(d.name == name){
|
if(d.name == copyObj.name){
|
||||||
dbo = d;
|
dbo = d;
|
||||||
break;
|
break;
|
||||||
}else{
|
}else{
|
||||||
@@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(!dbo){
|
if(!dbo){
|
||||||
result = findAddrAliasFromMap(city, region, address);
|
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||||
if(result.size()==1){
|
if(result.size()==1){
|
||||||
dbo = result[0];
|
dbo = result[0];
|
||||||
}else if(result && result.size()>1 && name){
|
}else if(result && result.size()>1 && name){
|
||||||
@@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(dbo){
|
||||||
|
return getFilerMatchDistrict(dbo, copyObj)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return dbo;
|
return dbo;
|
||||||
}
|
}
|
||||||
@@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
if(arrays[0] && arrays.size()==2){
|
if(arrays[0] && arrays.size()==2){
|
||||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ")
|
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ")
|
||||||
if(py1 == py2){
|
if(py1 == py2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def getOpenApiMatch(def city, def region, def name, def address){
|
def getOpenApiMatch(def data){
|
||||||
if(StringUtils.isNotEmpty(name)){
|
if(StringUtils.isNotEmpty(data.name)){
|
||||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false)
|
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
|
||||||
if(jsonObject && jsonObject.credit>0 ){
|
if(jsonObject && jsonObject.credit>0 ){
|
||||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit]
|
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
|
||||||
|
if(dbo){
|
||||||
|
return getFilerMatchDistrict(dbo, data)
|
||||||
|
}
|
||||||
|
return dbo
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def findAllNoMatchData(){
|
def findAllNoMatchData(){
|
||||||
DBObject qdata = new BasicDBObject();
|
DBObject qdata = new BasicDBObject();
|
||||||
qdata.put("s_date", SHORT_DATE)
|
qdata.put("s_date", SHORT_DATE)
|
||||||
|
// qdata.put("city","北京")
|
||||||
|
// qdata.put("source","安居客")
|
||||||
|
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
|
||||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||||
def ret = []
|
def ret = []
|
||||||
try{
|
try{
|
||||||
@@ -948,9 +1088,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def arrays = k.split("###")
|
def arrays = k.split("###")
|
||||||
if(arrays[0] && arrays[0]==keyword){
|
if(arrays[0] && arrays[0]==keyword){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:region, name:arrays[0]]
|
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -962,9 +1102,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def arrays = k.split("###")
|
def arrays = k.split("###")
|
||||||
if(arrays[0] && arrays[0]==keyword){
|
if(arrays[0] && arrays[0]==keyword){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:region, name:arrays[0]]
|
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -990,18 +1130,18 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def alias = v.alias;
|
def alias = v.alias;
|
||||||
if(alias.contains(keyword)){
|
if(alias.contains(keyword)){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:region, name:arrays[0]]
|
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else if(v && v.mapping_alias){
|
}else if(v && v.mapping_alias){
|
||||||
def mappingAlias = v.mapping_alias
|
def mappingAlias = v.mapping_alias
|
||||||
if(mappingAlias.contains(keyword)){
|
if(mappingAlias.contains(keyword)){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:region, name:arrays[0]]
|
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1016,18 +1156,18 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def alias = v.alias;
|
def alias = v.alias;
|
||||||
if(alias.contains(keyword)){
|
if(alias.contains(keyword)){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:r, name:arrays[0]]
|
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else if(v && v.mapping_alias){
|
}else if(v && v.mapping_alias){
|
||||||
def mappingAlias = v.mapping_alias
|
def mappingAlias = v.mapping_alias
|
||||||
if(mappingAlias.contains(keyword)){
|
if(mappingAlias.contains(keyword)){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:r, name:arrays[0]]
|
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1052,9 +1192,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def alias = v.addr_alias;
|
def alias = v.addr_alias;
|
||||||
if(alias.contains(keyword)){
|
if(alias.contains(keyword)){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:region, name:arrays[0]]
|
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1068,9 +1208,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def alias = v.addr_alias;
|
def alias = v.addr_alias;
|
||||||
if(alias.contains(keyword)){
|
if(alias.contains(keyword)){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:r, name:arrays[0]]
|
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1093,9 +1233,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def arrays = k.split("###")
|
def arrays = k.split("###")
|
||||||
if(arrays && arrays.size()==2 && arrays[1]==keyword){
|
if(arrays && arrays.size()==2 && arrays[1]==keyword){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:region, name:arrays[0]]
|
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1106,9 +1246,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def arrays = k.split("###")
|
def arrays = k.split("###")
|
||||||
if(arrays && arrays.size()==2 && arrays[1]==keyword){
|
if(arrays && arrays.size()==2 && arrays[1]==keyword){
|
||||||
if(arrays.size()>=2){
|
if(arrays.size()>=2){
|
||||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
|
||||||
}else{
|
}else{
|
||||||
districts << [city:city, region:r, name:arrays[0]]
|
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-5
@@ -24,8 +24,8 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||||
|
|
||||||
// "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
|
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
|
||||||
def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
|
def esSources = ["中原地产":"中原地产"]
|
||||||
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
@@ -57,7 +57,7 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
def names = new HashSet()
|
def names = new HashSet()
|
||||||
def rets = []
|
def rets = []
|
||||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
|
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
|
||||||
def s = d.city +"#"+d.region+"#"+d.name
|
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
|
||||||
if(!names.contains(s)){
|
if(!names.contains(s)){
|
||||||
names << s
|
names << s
|
||||||
def data = new BasicDBObject()
|
def data = new BasicDBObject()
|
||||||
@@ -67,6 +67,7 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
data.put("address", d.address)
|
data.put("address", d.address)
|
||||||
data.put("avg_price", d.avg_price)
|
data.put("avg_price", d.avg_price)
|
||||||
data.put("source", mgSource)
|
data.put("source", mgSource)
|
||||||
|
data.put("estate_type",d.property_type)
|
||||||
data.put("s_date", SHORT_DATE)
|
data.put("s_date", SHORT_DATE)
|
||||||
data.put("url", d.url)
|
data.put("url", d.url)
|
||||||
data.put("c_date", new Date())
|
data.put("c_date", new Date())
|
||||||
@@ -144,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
|
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
|
||||||
}else{
|
}else{
|
||||||
for(def r : rets){
|
for(def r : rets){
|
||||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source)
|
||||||
if(obj){
|
if(obj){
|
||||||
updateIndex++
|
updateIndex++
|
||||||
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||||
@@ -152,6 +153,7 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
|
|
||||||
def udata = new BasicDBObject()
|
def udata = new BasicDBObject()
|
||||||
udata.put("avg_price", r.avg_price)
|
udata.put("avg_price", r.avg_price)
|
||||||
|
udata.put("estate_type", r.estate_type)
|
||||||
udata.put("m_date",new Date())
|
udata.put("m_date",new Date())
|
||||||
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
|
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
|
||||||
|
|
||||||
@@ -181,13 +183,16 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def findIsExists(def city, def region, def name, def sDate, def source){
|
def findIsExists(def city, def region, def name, estate_type, def sDate, def source){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("source", source)
|
q.put("source", source)
|
||||||
q.put("s_date", sDate)
|
q.put("s_date", sDate)
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
q.put("region", region)
|
q.put("region", region)
|
||||||
q.put("name", name)
|
q.put("name", name)
|
||||||
|
if(estate_type){
|
||||||
|
q.put("estate_type",estate_type)
|
||||||
|
}
|
||||||
def obj = thirdPartyColl.findOne(q)
|
def obj = thirdPartyColl.findOne(q)
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user