From c83a6d13696ad2f8e2b88753369663789ce2bcf5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 Feb 2023 10:54:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=8E=BB=E9=87=8D=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 102 +++++++++++++----- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 007fc2b86..74740ef87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -107,6 +108,33 @@ class BigdataForsaleBatch extends BaseBatch{ def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ + def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate) + println city + "(原)房源总量:"+retMap.size() + + def allDatas = retMap.values() + allDatas = repeatHouseId(allDatas) + println city + "(新)房源总量:"+allDatas.size() + + + def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" + } + + + // def allDatas = createSellDatas(city, retMap.values()); + + def headers = [] +// if(index == 1){ +// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] +// } + FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); + println "txt文件生成->手动导入到Oracle " + + saveToPgSql(allDatas) + } + + def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() @@ -172,23 +200,38 @@ class BigdataForsaleBatch extends BaseBatch{ }) } latch.await() + return retMap + } - def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" + + def repeatHouseId(allDatas){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(allDatas.size()) + def rets = new CopyOnWriteArrayList() + for(def i=0;i手动导入到Oracle " - - saveToPgSql(allDatas) + latch.await() + return rets } @@ -440,20 +483,21 @@ class BigdataForsaleBatch extends BaseBatch{ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) - if(rets1){ - def newRets = [] - def cityCode = cityCodeDataMap.get(city) - def houseIds = getAllHouseIds(commId, cityCode) - for(def r : rets1){ - if(!houseIds.contains(r.houseId)){ - newRets << r - }else{ - println r.houseId +" 已存在" - } - } - return newRets - } - return []; +// if(rets1){ +// def newRets = [] +// def cityCode = cityCodeDataMap.get(city) +// def houseIds = getAllHouseIds(commId, cityCode) +// for(def r : rets1){ +// if(!houseIds.contains(r.houseId)){ +// newRets << r +// }else{ +// println r.houseId +" 已存在" +// } +// } +// return newRets +// } +// return []; + return rets1; } def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 0bab915c0..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 94a13d8e8..e82b8bc45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } }