+41
-25
@@ -80,12 +80,11 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
for(def city : citys){
|
||||
index++
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
}
|
||||
@@ -114,8 +113,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
println city + "(原)房源总量:"+retMap.size()
|
||||
|
||||
def allDatas = retMap.values()
|
||||
allDatas = repeatHouseId(allDatas)
|
||||
println city + "(新)房源总量:"+allDatas.size()
|
||||
// // allDatas = repeatHouseId(allDatas)
|
||||
// println city + "(新)房源总量:"+allDatas.size()
|
||||
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
|
||||
@@ -140,6 +139,9 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
def houseIds = findAllHouseIdsByCity(cityCodeDataMap.get(city))
|
||||
println city +" ${city} 库已存在房源:"+districts.size()
|
||||
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
@@ -168,23 +170,27 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
m.district_id = commId
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
if(!houseIds.contains(m.houseId)){
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
m.district_id = commId
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
}else{
|
||||
println city +" "+m.houseId+" 已存在"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+rMap.region+" 没有找到对应的区域"
|
||||
@@ -482,7 +488,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
|
||||
|
||||
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
// if(rets1){
|
||||
@@ -1090,6 +1096,16 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
def findAllHouseIdsByCity(def cityCode){
|
||||
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"'";
|
||||
|
||||
def houseIds = new HashSet()
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
houseIds << it.FORSALEID
|
||||
});
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
|
||||
def isExistFromHouse(def houseId){
|
||||
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
|
||||
@@ -1109,7 +1125,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
|
||||
+2
-2
@@ -127,12 +127,12 @@ class CcbCommonUtils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q).sort(new BasicDBObject("guiding_date",-1))
|
||||
def rets = []
|
||||
def sets = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
|
||||
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
|
||||
def key = d.city +"##"+d.region+"##"+d.name
|
||||
if(!sets.contains(key)){
|
||||
rets << d
|
||||
sets << key
|
||||
|
||||
Reference in New Issue
Block a user