房源导入

This commit is contained in:
2023-03-07 14:54:02 +08:00
parent 6e947523c0
commit 90733d071b
2 changed files with 43 additions and 27 deletions
@@ -80,12 +80,11 @@ class BigdataForsaleBatch extends BaseBatch{
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -114,8 +113,8 @@ class BigdataForsaleBatch extends BaseBatch{
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
allDatas = repeatHouseId(allDatas)
println city + "(新)房源总量:"+allDatas.size()
// // allDatas = repeatHouseId(allDatas)
// println city + "(新)房源总量:"+allDatas.size()
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
@@ -140,6 +139,9 @@ class BigdataForsaleBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def houseIds = findAllHouseIdsByCity(cityCodeDataMap.get(city))
println city +" ${city} 库已存在房源:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
@@ -168,23 +170,27 @@ class BigdataForsaleBatch extends BaseBatch{
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
if(!houseIds.contains(m.houseId)){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}else{
println city +" "+m.houseId+" 已存在"
}
}
}else{
println city +" "+rMap.region+" 没有找到对应的区域"
@@ -482,7 +488,7 @@ class BigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
@@ -1090,6 +1096,16 @@ class BigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(def cityCode){
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"'";
def houseIds = new HashSet()
sqlInstance.eachRow(sql,{it->
houseIds << it.FORSALEID
});
return houseIds;
}
def isExistFromHouse(def houseId){
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
@@ -1109,7 +1125,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
}
r << null
r << '2023-02-28'
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
@@ -127,12 +127,12 @@ class CcbCommonUtils {
def q = new BasicDBObject()
q.put("city", city)
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def dbcursor = GUIDING_PRICE_COLL.find(q).sort(new BasicDBObject("guiding_date",-1))
def rets = []
def sets = new HashSet()
for(def d : dbcursor){
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
def key = d.city +"##"+d.region+"##"+d.name
if(!sets.contains(key)){
rets << d
sets << key