+73
-29
@@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -107,6 +108,33 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
|
||||
|
||||
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate)
|
||||
println city + "(原)房源总量:"+retMap.size()
|
||||
|
||||
def allDatas = retMap.values()
|
||||
allDatas = repeatHouseId(allDatas)
|
||||
println city + "(新)房源总量:"+allDatas.size()
|
||||
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
|
||||
}
|
||||
|
||||
|
||||
// def allDatas = createSellDatas(city, retMap.values());
|
||||
|
||||
def headers = []
|
||||
// if(index == 1){
|
||||
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
// }
|
||||
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
|
||||
println "txt文件生成->手动导入到Oracle "
|
||||
|
||||
saveToPgSql(allDatas)
|
||||
}
|
||||
|
||||
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
@@ -172,23 +200,38 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
|
||||
|
||||
def repeatHouseId(allDatas){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(allDatas.size())
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<allDatas.size();i++){
|
||||
final def houseInfo = allDatas[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = isExistFromHouse(houseInfo[1])
|
||||
if(count){
|
||||
println houseInfo[1]+" 已存在"
|
||||
}else{
|
||||
rets << houseInfo
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
// def allDatas = createSellDatas(city, retMap.values());
|
||||
def allDatas = retMap.values()
|
||||
def headers = []
|
||||
// if(index == 1){
|
||||
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
// }
|
||||
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
|
||||
println "txt文件生成->手动导入到Oracle "
|
||||
|
||||
saveToPgSql(allDatas)
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
@@ -440,20 +483,21 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
if(rets1){
|
||||
def newRets = []
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def houseIds = getAllHouseIds(commId, cityCode)
|
||||
for(def r : rets1){
|
||||
if(!houseIds.contains(r.houseId)){
|
||||
newRets << r
|
||||
}else{
|
||||
println r.houseId +" 已存在"
|
||||
}
|
||||
}
|
||||
return newRets
|
||||
}
|
||||
return [];
|
||||
// if(rets1){
|
||||
// def newRets = []
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
// def houseIds = getAllHouseIds(commId, cityCode)
|
||||
// for(def r : rets1){
|
||||
// if(!houseIds.contains(r.houseId)){
|
||||
// newRets << r
|
||||
// }else{
|
||||
// println r.houseId +" 已存在"
|
||||
// }
|
||||
// }
|
||||
// return newRets
|
||||
// }
|
||||
// return [];
|
||||
return rets1;
|
||||
}
|
||||
|
||||
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
+2
-2
@@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("福州")){
|
||||
// if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+1
-1
@@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user