From 6a6549756c6fab7216eda85a3f8880e0e04ccc67 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Nov 2024 15:30:31 +0800 Subject: [PATCH] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/ThirdPriceMappingBatch.groovy | 7 +++- .../thirdPriceCrawl/DataStandardBatch.groovy | 37 ++++++------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 5acefdb73..a6180d973 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -42,14 +42,19 @@ class ThirdPriceMappingBatch extends BaseBatch{ // //将新增数据同步过来 // addNemCommunity() //// //接口进行匹配 -// match() + match() addOldMatchResult() } def match(){ def citys = CommonUtils.findAllCity() + def index = 0 for(def c : citys){ + index ++ + if(index < 206){ + continue + } def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false)) def noMatchDatas = thirdSeaweedMappingColl.find(query).toList() matchByCity(c, noMatchDatas) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 25e01d8a5..b57cdd659 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -7,21 +7,8 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.NameProcess import com.fangjia.script.gj.district.SeaweedDataMatchProcess -import com.fangjia.script.linxsh.util.CommonUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.CosineSimilarAlgorithm -import com.fangjia.script.util.DataUtil -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DB -import com.mongodb.DBCollection -import com.mongodb.DBObject -import com.mongodb.QueryOperators -import net.sf.json.JSONObject +import com.fangjia.script.util.* +import com.mongodb.* import org.apache.commons.lang3.SerializationUtils import org.apache.commons.lang3.StringUtils import org.apache.tools.ant.util.DateUtils @@ -31,9 +18,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger -import java.util.regex.Matcher import java.util.regex.Pattern - /** * 第三方数据标准化处理 * 成功type说明 @@ -689,14 +674,15 @@ class DataStandardBatch extends BaseBatch{ def pre = (data.avg_price-price)*100/price if(pre>=-20 && pre<=20){ return dbo; - }else{ - def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) - if(!ret){ - data.put("msg","房源价格过滤") - THIRD_PRICE_FILTER_COLL.save(data) - } - return null } +// else{ +// def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) +// if(!ret){ +// data.put("msg","房源价格过滤") +// THIRD_PRICE_FILTER_COLL.save(data) +// } +// return null +// } } } return dbo @@ -934,7 +920,8 @@ class DataStandardBatch extends BaseBatch{ DBObject qdata = new BasicDBObject(); qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE)) // qdata.put("city","上海") -// qdata.put("name","白金瀚宫") +// qdata.put("name","绿地云天坊") +// qdata.put("status", 3) // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));