From d983a7992784feb3c7f7e56d852811c1b8cefb05 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 24 Sep 2024 16:08:57 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=8A=A0=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 4bb2aff5a..668fcbb74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -70,6 +70,10 @@ class SynchDistrictPrice extends BaseBatch{ data.put("estate_type",d.property_type) data.put("s_date", SHORT_DATE) data.put("url", d.url) + data.put("id",d.id) + data.put("ownership_type",d.ownership_type) + data.put("property_type",d.property_type) + data.put("build_type",d.build_type) data.put("c_date", new Date()) data.put("m_date", new Date()) rets << data From 5dc42248470276e3fe4e1388e35fa4ae44133ce3 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 24 Sep 2024 17:49:42 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=9D=87?= =?UTF-8?q?=E4=BB=B7=E5=8C=B9=E9=85=8D=EF=BC=8C=E7=94=A8=E6=96=B0=E5=8C=B9?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/ThirdPriceMappingBatch.groovy | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy new file mode 100644 index 000000000..9c5cccb47 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -0,0 +1,150 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 第三方小区与我们小区映射关系表 + */ +@BatchOwner(Watcher.GaoJuan) +class ThirdPriceMappingBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + static main(args) { + runBatch(new ThirdPriceMappingBatch(), args); + } + + @Override + public void process(Object args) { + //将新增数据同步过来 + addNemCommunity() + //接口进行匹配 + match() + } + + def match(){ + def citys = CommonUtils.findAllCity() + for(def c : citys){ + def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false)) + def noMatchDatas = thirdSeaweedMappingColl.find(query).toList() + matchByCity(c, noMatchDatas) + } + + } + + def matchByCity(def c, def noMatchDatas){ + def totalCount = new AtomicLong() + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(noMatchDatas.size()) + def total = noMatchDatas.size() + for(def i=0;i + udata.put(k,v) + } + udata.put("m_date", new Date()) + thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) + }else{ + sd.put("m_date", new Date()) + thirdSeaweedMappingColl.save(sd) + addCount.addAndGet(1) + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + return addCount.get(); + } +} From 3854a5589f2d221376c226340c3d3ec618f7e275 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 24 Sep 2024 18:04:16 +0800 Subject: [PATCH 3/4] update --- .../fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 9c5cccb47..dd8f3e1a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -118,7 +118,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ try{ sd.put("district", sd.region) sd.removeField("region") - sd.removeField("url") +// sd.removeField("url") sd.removeField("s_date") totalCount.addAndGet(1) def query = new BasicDBObject("url", sd.url) From 60ed1fc372f5f02b6a9261d74fa7c77eb6b91ee3 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 25 Sep 2024 11:14:42 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E5=8C=B9=E9=85=8D19=E6=8D=A293?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/ThirdPriceMappingBatch.groovy | 72 +++++++++++++++++-- .../gj/district/FangjiaClientUtils.groovy | 4 +- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index dd8f3e1a6..2d7515aa7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -3,12 +3,15 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.bson.types.ObjectId import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -28,6 +31,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static main(args) { @@ -36,10 +42,12 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { - //将新增数据同步过来 - addNemCommunity() - //接口进行匹配 +// //将新增数据同步过来 +// addNemCommunity() +// //接口进行匹配 match() + +// addOldMatchResult() } def match(){ @@ -63,7 +71,11 @@ class ThirdPriceMappingBatch extends BaseBatch{ void run() { try{ addCount.addAndGet(1) - def address = nmd.city + nmd.district + nmd.name + def address = nmd.city + if(nmd.district){ + address = address + nmd.district + } + address = address + nmd.name def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true) if(jsonObject) { def udata = new BasicDBObject() @@ -147,4 +159,56 @@ class ThirdPriceMappingBatch extends BaseBatch{ latch.await() return addCount.get(); } + + def addOldMatchResult(){ + def citys = ["上海"]//CommonUtils.findAllCity() + def keys = new BasicDBObject() + keys.put("_id", 1) + + for(def c : citys){ + def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList() + println c +" 待关联数据:"+datas.size() + findResultFromDB(c, datas) + println c + " 关联数据结束 "+DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd HH:mm:ss") + } + } + + def findResultFromDB(city, datas){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + def total = datas.size() + for(def i=0;i