diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 8c895b5c0..09f80b9da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -12,7 +12,7 @@ class FileUtils { OutputStream out = new FileOutputStream(fileName); def sb = new StringBuffer(); - sb.append(headers.join("@|@")) + sb.append(headers.join("|@|")) sb.append(System.getProperty("line.separator")); out.write(sb.toString().getBytes(charset)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 27f1870bb..7027cf373 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -76,6 +76,8 @@ class HouseExport { } def exportSellCSV(city, districts){ + + final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() def datas = [] @@ -124,11 +126,16 @@ class HouseExport { }) } latch.await() - println " 去重后:"+retMap.values() - def allDatas = createSellDatas(retMap.values()); + def allDatas = createSellDatas(retMap); def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"] - batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forsale_${cityCodeMap[city]}.csv",headers, "utf-8"); - println city +" [sell]生成完成!" + + def count = 1; + for(def allData : allDatas){ + FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_2017_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); + println city +" [sell-${count}]生成完成!" + count ++; + } + } @@ -184,17 +191,19 @@ class HouseExport { }) } latch.await() - println " 去重后:"+retMap.values() - def allDatas = createRentDatas(retMap.values()); + + def allDatas = createRentDatas(retMap); def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"] - batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8"); println city +" [rent]生成完成!" } - def createSellDatas(def list){ + def createSellDatas(def map){ + def datas1 = [] + def datas2 = [] def datas = [] - for(def m : list){ + map.each{k,m-> def data = [] data << m.city_code data << m.houseId @@ -248,15 +257,26 @@ class HouseExport { data << m.m_date==null?"":m.m_date data << m.source - datas << data + if(datas1.size()>2000000){ + datas2 << data + }else{ + datas1 << data + } + } + if(datas1.size()>0){ + datas << datas1 + } + + if(datas2.size()>0){ + datas << datas2 } return datas ; } - def createRentDatas(def list){ + def createRentDatas(def map){ def datas = [] - for(def m : list){ + map.each{k,m-> def data = [] data << m.city_code data << m.houseId