diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2efd484b..ef175956f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -46,16 +47,19 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" 小区量:"+dists.size() + + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) + rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) println city +" "+rets.size() // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } - def fileName = "/home/goojia/ccb/bigdata_community.dat" + def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" } def allDatas = [] @@ -66,8 +70,27 @@ class BigdataCommunityBatch extends BaseBatch{ // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } + def addDeleteDist(dists, ccbMaps, rets){ + def fangjiaDists = [] + for(def d : dists){ + fangjiaDists << d._id.toString() + } + println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size() + def ccbDists = ccbMaps.keySet() + for(def ccbId : ccbDists){ + if(!fangjiaDists.contains(ccbId)){ + def datas = ccbMaps.get(ccbId) + datas[datas.size()-1] = 1 + rets << datas + } + } + println "添加删除状态小区:"+rets.size() + return rets + } + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ - def rets = Collections.synchronizedList(new ArrayList()); + //def rets = Collections.synchronizedList(new ArrayList()); + def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) for(def i=0;i + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + map.put(r.COMMID, data) + }); + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 98eca9153..4a8801a5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -47,12 +47,22 @@ class SFTP { * @param sftp */ public void upload(String directory, String uploadFile, ChannelSftp sftp) { + FileInputStream stream = null try { sftp.cd(directory); File file=new File(uploadFile); - sftp.put(new FileInputStream(file), file.getName()); + stream = new FileInputStream(file) + sftp.put(stream, file.getName()); } catch (Exception e) { e.printStackTrace(); + } finally { + if(stream!=null){ + stream.close(); + } +// if (sftp != null) { +// sftp.disconnect(); +// sftp.exit(); +// } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a8b9a0427..0ba834ad5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,9 +15,9 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", - "anjuke.com":"安居客","cityhouse.cn":"城市房产"] -// def sourceMap = ["lianjia.com":"链家"] +// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", +// "anjuke.com":"安居客","cityhouse.cn":"城市房产"] + def sourceMap = ["zhuge.com":"诸葛找房"] static main(args) { def task = new SynchDistrictPrice() task.run()