房源去重修改

This commit is contained in:
2023-02-19 10:54:55 +08:00
parent 05673e6b2c
commit c83a6d1369
3 changed files with 76 additions and 32 deletions
@@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -107,6 +108,33 @@ class BigdataForsaleBatch extends BaseBatch{
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
allDatas = repeatHouseId(allDatas)
println city + "(新)房源总量:"+allDatas.size()
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
}
// def allDatas = createSellDatas(city, retMap.values());
def headers = []
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成->手动导入到Oracle "
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
@@ -172,23 +200,38 @@ class BigdataForsaleBatch extends BaseBatch{
})
}
latch.await()
return retMap
}
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
def repeatHouseId(allDatas){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(allDatas.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<allDatas.size();i++){
final def houseInfo = allDatas[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = isExistFromHouse(houseInfo[1])
if(count){
println houseInfo[1]+" 已存在"
}else{
rets << houseInfo
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = []
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成->手动导入到Oracle "
saveToPgSql(allDatas)
latch.await()
return rets
}
@@ -440,20 +483,21 @@ class BigdataForsaleBatch extends BaseBatch{
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
if(rets1){
def newRets = []
def cityCode = cityCodeDataMap.get(city)
def houseIds = getAllHouseIds(commId, cityCode)
for(def r : rets1){
if(!houseIds.contains(r.houseId)){
newRets << r
}else{
println r.houseId +" 已存在"
}
}
return newRets
}
return [];
// if(rets1){
// def newRets = []
// def cityCode = cityCodeDataMap.get(city)
// def houseIds = getAllHouseIds(commId, cityCode)
// for(def r : rets1){
// if(!houseIds.contains(r.houseId)){
// newRets << r
// }else{
// println r.houseId +" 已存在"
// }
// }
// return newRets
// }
// return [];
return rets1;
}
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
@@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("福州")){
// if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
}
// }
}
}
return list
@@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}