diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 8566cf085..f928400f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -144,16 +144,17 @@ class BigdataDealBatch extends BaseBatch{ def street = "" def streetNumber = "" if(trade.address){ - if(trade.address.toString().endsWith("Ū")){ + if(trade.address.toString().contains("Ū")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+Ū")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) streetNumber = number } - }else{ + } + if(!street){ def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) if(jsonObj && jsonObj.searchAddress){ - if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("Ū")){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("Ū")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+Ū")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) @@ -170,7 +171,7 @@ class BigdataDealBatch extends BaseBatch{ if(!street){ if(obj.address){ - if(obj.address.toString().endsWith("Ū")){ + if(obj.address.toString().contains("Ū")){ String number = DataUtil.match(obj.address, Pattern.compile("\\d+Ū")) if(number){ street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) @@ -282,7 +283,7 @@ class BigdataDealBatch extends BaseBatch{ } if(trade.address){ - if(trade.address.toString().endsWith("Ū")){ + if(trade.address.toString().contains("Ū")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+Ū")) if(number){ ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number)) @@ -420,6 +421,9 @@ class BigdataDealBatch extends BaseBatch{ if(dist.city=="ÉϺ£"){ startPrice = (int)(price - price*0.1) endPrice = (int)(price*0.5 + price) + }else if(dist.city=="º¼ÖÝ"){ + startPrice = (int)(price - price*0.2) + endPrice = (int)(price*0.5 + price) }else{ startPrice = (int)(price - price*0.5) endPrice = (int)(price*0.5 + price) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 797641971..d10aab75c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["¶«Ý¸","ÖÐɽ","ÎIJý","±£Í¤","ÁêË®"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 999d4b347..0b413551a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -187,7 +187,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "ʯȪÁù´å") +// q.put("name", "À³Ð˹«Ô¢") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["ÆÕͨסլ","ÉÌס"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index 15d2fe3d4..c71f088de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -2,6 +2,11 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.util.HttpRequest import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import net.sf.json.JSONObject import org.elasticsearch.xpack.sql.jdbc.EsDataSource @@ -26,16 +31,112 @@ class UpdateHouseRegion { } public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + static main(args) { def task = new UpdateHouseRegion() - task.run() +// task.updateSeawaterRegion() +// task.updateDealRegion() + task.updateThirdPartyRegion() } - def run(){ + + def updateThirdPartyRegion(){ + def city = "º¼ÖÝ" + def rets = findThirdPartyPrice(city) + batchThirdPartyUpdate(city, rets); + + } + def findThirdPartyPrice(city){ + def q = new BasicDBObject() + q.put("city",city) + q.put("s_date",new BasicDBObject('$gte', 202101)) + def dbcursor = thirdPartyColl.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + + def updateDealRegion(){ + def city = "º¼ÖÝ" + def q = new BasicDBObject() + q.put("city", city) + q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01"))); + def cursor = DEAL_PRICE_COLL.find(q) + def index = 0 + def updateIndex = 0 + for(def obj : cursor){ + index++ + def region = obj.region + def name = obj.district_name + def rets = CommonDatautils.findDistrictsByName(city, name, "district") + if (!rets) { + rets = CommonDatautils.findDistrictByAlias(city, null, name, "district") + } + + if (rets.size() == 1) { + def dist = rets[0] + if (region != dist.region) { + updateIndex++ + def udata = new BasicDBObject() + udata.put("region", dist.region) + DEAL_PRICE_COLL.update(new BasicDBObject('_id', obj.get("_id")), new BasicDBObject('$set', udata)) + println updateIndex+" "+index+" "+region+" "+dist.region + } + } + } + + + } + def updateSeawaterRegion(){ def city = "º¼ÖÝ" def rets = findHouseSource(city, 20190701, 20191231) batchUpdate(city, rets); } + + def batchThirdPartyUpdate(def city, def thirdDatas){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = thirdDatas.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" µÚÈý·½×ÜÁ¿£º"+totalCount + for(def i=0;i