diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index fafe61ba9..2439e06d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -51,7 +51,7 @@ class BigdataCommunityBatch extends BaseBatch{ index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) @@ -175,25 +175,31 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa101" ret << "" if(dist.address){ - def jsonObj = null - if(dist.address){ - jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) - } - if(jsonObj && jsonObj.result && jsonObj.result.street){ - if(jsonObj.result.street){ - ret << jsonObj.result.street - }else{ - ret << "" - } - - if(jsonObj.result.streetNumber){ - ret << jsonObj.result.streetNumber + if(dist.street){ + ret << dist.street + if(dist.street_number){ + ret << dist.street_number }else{ ret << "" } }else{ - ret << "" - ret << "" + def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) + if(jsonObj && jsonObj.result && jsonObj.result.street){ + if(jsonObj.result.street){ + ret << jsonObj.result.street + }else{ + ret << "" + } + + if(jsonObj.result.streetNumber){ + ret << jsonObj.result.streetNumber + }else{ + ret << "" + } + }else{ + ret << "" + ret << "" + } } }else{ ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 789603d29..a1a81702d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -33,8 +33,8 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ -// def startDate = 202206 -// def endDate = 202208 + def startDate = 202206 + def endDate = 202208 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) @@ -58,11 +58,12 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } + println "curDate:"+curDate // for(def city : citys){ //// if(!ccbCitys.contains(city)){ // addDealPriceMatch(city); @@ -110,9 +111,9 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) -// def lianjiaPriceMap = findThirdPartyPrice("上海") + def lianjiaPriceMap = findThirdPartyPrice("上海") def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') - def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) +// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) for(def i=0;i + if(d['A']){ + def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(dbo){ + def udate = new BasicDBObject() + def street = null + def address = "" + if(d['E']){ + street = d['E'] + udate.put("street", street) + address = street + } + def streetNum = null + if(d['F']){ + streetNum = d['F'] + udate.put("street_number", streetNum) + address = address + streetNum + } + + if(dbo.address!=address){ + udate.put("address", address) + } + if(udate){ + udate.put("m_date", new Date()) + DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate)) + } + + } + } + + }) } def sameAddr() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy new file mode 100644 index 000000000..1656648bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -0,0 +1,15 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch + +class BigdataCitypriceBatch extends BaseBatch{ + + static main(args) { + runBatch(new BigdataCitypriceBatch(), args) + } + + @Override + void process(Object args) { + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index b8116db22..77d788a46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // }) // } - addBigdataDistrictFJW() +// addBigdataDistrictFJW() -// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); + def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); + + def index = 0 + for (def city : citys) { + index++ + def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) // -// def index = 0 -// for (def city : citys) { -// index++ -// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -//// -//// for(int i=0;i<10;i++){ -//// if(districts.size()==0){ -//// Thread.sleep(1000*60) -//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -//// }else{ -//// break -//// } -//// } -// -// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) -// -// for(def dist : districts){ -// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) -// if(obj){ -// if(obj.alias && obj.alias.size()>0){ -// dist.commalias = obj.alias.get(0) -// }else{ -// dist.commalias = "" -// } -// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) -// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) -// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); -// dist.houseuse = "11" -// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) -// dist.compid = new ObjectId() //物业公司id -// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) +// for(int i=0;i<10;i++){ +// if(districts.size()==0){ +// Thread.sleep(1000*60) +// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +// }else{ +// break // } // } -// println index +" "+city +" 小区加载完成 "+districts.size() -// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 1.小区表完成" -// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 2.小区建筑结构表" -// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 3.小区房屋类型表" -// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 4.小区建筑年代表" -// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 5.小区费用表" -// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 6.小区房屋用途表" -// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) -// println index +" "+ city + " 7.地理位置表" -// -//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 -// -// } + + def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) + + for(def dist : districts){ + def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + if(obj){ + if(obj.alias && obj.alias.size()>0){ + dist.commalias = obj.alias.get(0) + }else{ + dist.commalias = "" + } + dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) + dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) + dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); + dist.houseuse = CcbCodeUtils.getUseTypeCode(obj.estate_type2)//"11" + dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) + dist.compid = new ObjectId() //物业公司id + dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) + } + } + println index +" "+city +" 小区加载完成 "+districts.size() + addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 1.小区表完成" + addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 2.小区建筑结构表" + addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 3.小区房屋类型表" + addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 4.小区建筑年代表" + addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 5.小区费用表" + addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 6.小区房屋用途表" + addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) + println index +" "+ city + " 7.地理位置表" + +// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 + + } } def addBigdataCommunityGJRH(def districts, def cityid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 517bfef5c..8d77112a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -31,6 +31,8 @@ class DistrictPriceExportBatch extends BaseBatch{ public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) @@ -41,18 +43,8 @@ class DistrictPriceExportBatch extends BaseBatch{ def headers = ["id","comm_id","city","district","name","avg_price","year_month"] - def citys = CommonDatautils.findAllCity() - citys.remove("福州"); - citys.add(0, "福州") - - def fz_datas = [] - for(def city : citys){ - def ret = createDatas("福州") - saveToDB(ret) - if(city == "福州"){ - fz_datas = ret - } - } + def ret = createNewDatas("福州") + saveToDB(ret) def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" @@ -60,7 +52,7 @@ class DistrictPriceExportBatch extends BaseBatch{ fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv" dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512" } - def sb = appendResult(headers, fz_datas) + def sb = appendResult(headers, ret) DataHandleUtils.outPutFile(fileName, sb, charset); FzFtpUtils.ftpUpload(fileName) //生成加密数据文件并上传 @@ -106,16 +98,50 @@ class DistrictPriceExportBatch extends BaseBatch{ return rets } + + def createNewDatas(def city){ + def districts = getFzDistricts(city) + def curPriceMap = findFzDistrictSellStat("福州",curDate) + def result = [] + for(def rMap : districts){ + def curPrice = 0 + def dbo = null + if(curPriceMap.containsKey(rMap.get("_id").toString())){ + dbo = curPriceMap.get(rMap.get("_id").toString()) + curPrice = dbo.avg_price + }else{ + curPrice = specialDistsPriceMap.get(rMap.name) + if(curPrice){ + def randMarkup = nextDouble(0, 1) + curPrice = (int)(curPrice*randMarkup/100 + curPrice) + } + } + def m = [] + if(curPrice && dbo){ + m << dbo.get("_id").toString()//均价ID + }else{ + m << new ObjectId().toString() + } + m << rMap.get("_id").toString()//楼盘ID + m << rMap.city + m << rMap.region + m << rMap.name + if(curPrice){ + m << curPrice + }else{ + m << 0 + } + m << curDate.toInteger(); + result << m + } + return result + } + def createDatas(def city){ // def districts = CommonDatautils.findAllDistricts(city, "district", null); // def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); // districts.addAll(removeDists) - def districts = [] - if(city == "福州"){ - districts = getFzDistricts(city) - }else{ - districts = CommonDatautils.findAllDistricts(city, "district", null); - } + def districts = getFzDistricts(city) def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -128,38 +154,27 @@ class DistrictPriceExportBatch extends BaseBatch{ // def key = row[2] + "##" + row[3] + "##" + row[4] // map2.put(key, row[5]) // } - def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate) + def result = [] + for(def rMap : districts){ - def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) + def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, curDate) def curPrice = 0 if(dbo){ def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString()) curPrice = dbo.get("avg_price") if(lastMonthPrice && curPrice && dbo.value_type!=1){ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice - if(city == "福州"){ - if(Math.abs(markup)>3){ - def randMarkup = nextDouble(0, 3) - if(markup < 0){ - markup = randMarkup*(-1) - }else{ - markup = randMarkup - } - curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) - } - }else{ - if(Math.abs(markup)>10){ - def randMarkup = nextDouble(0, 10) - if(markup < 0){ - markup = randMarkup*(-10) - }else{ - markup = randMarkup - } - curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) + if(Math.abs(markup)>3){ + def randMarkup = nextDouble(0, 3) + if(markup < 0){ + markup = randMarkup*(-1) + }else{ + markup = randMarkup } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } }else{ @@ -190,6 +205,17 @@ class DistrictPriceExportBatch extends BaseBatch{ return result } + def findFzDistrictSellStat(city,lastMonthDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("date", lastMonthDate) + def priceMap = [:] + DISTRICT_SELL_STAT.find(q).each{ + priceMap.put(it.commid, it) + } + return priceMap + } + def findFzDistrictStat(city,lastMonthDate){ def q = new BasicDBObject() @@ -256,7 +282,6 @@ class DistrictPriceExportBatch extends BaseBatch{ udata.put("avg_price", r[5]) FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata)) }else{ - def udata = new BasicDBObject() def o = new BasicDBObject() println r[0] o.put("_id", new ObjectId(r[0])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy new file mode 100644 index 000000000..a11d0ce17 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -0,0 +1,321 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.text.DecimalFormat +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +@BatchOwner(Watcher.GaoJuan) +class DistrictSellStatBatch extends BaseBatch { + + static DBCollection FZ_DISTRICT_STAT = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + + static DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + + public static void main(String[] args) { + runBatch(new DistrictSellStatBatch(), args); + } + + @Override + void process(Object args) { + calcMonthlyPrice() + } + + + def getMomDate(int date){ + + Calendar calendar = Calendar.getInstance() + calendar.setTime(Date.parse("yyyyMM", String.valueOf(date))) + calendar.add(Calendar.MONTH, -1) + return calendar.getTime().format("yyyyMM").toInteger() + } + + def getYoyDate(int date){ + + Calendar calendar = Calendar.getInstance() + calendar.setTime(Date.parse("yyyyMM", String.valueOf(date))) + calendar.add(Calendar.YEAR, -1) + return calendar.getTime().format("yyyyMM").toInteger() + } + + def calcMonthlyPrice(){ + def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity() + def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) + + def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) + + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def monDate = getMomDate(curMonth) + def yoyDate = getYoyDate(curMonth) + + def index = 0 + for(def city : citys){ + + index++ + def districts = [] + if(city == "福州"){ + districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() + }else{ + districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) + } + + def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 + def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 + def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价 + def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价 + if(districts && weekendPriceMap){ + running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap) + } + println index+" "+city +" "+ (new Date()).toString() + } + } + + def running(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + districts.each{ dist-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + println totalCount.get()+" "+ dist.city+" "+dist.name + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } + + def running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def rets = new CopyOnWriteArrayList() + + districts.each{dist-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + def key = dist.region +"##" + dist.name + def weekendPriceObj = weekendPriceMap.get(key) + if(weekendPriceObj){ + def curPriceObj = curMonthlyPriceMap.get(key) + def lastMonthPriceObj = lastMonthlyPriceMap.get(key) + def lastYearPriceObj = lastYearPriceMap.get(key) + double mom = 0d + + if(lastMonthPriceObj) { + mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) + } + + def curPrice = null; + if(curPriceObj){ + curPrice = curPriceObj.avg_price + }else{ + curPrice = weekendPriceObj.avg_price + } + + if(weekendPriceObj.value_type!=1){ + if(dist.city == "福州"){ + if(Math.abs(mom)*100>3){ + def randMarkup = nextDouble(0, 3) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100, 4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) + } + }else{ + if(Math.abs(mom)*100>10){ + def randMarkup = nextDouble(0, 10) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100,4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100,4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) + } + } + } + + double yoy = 0d + if(lastYearPriceObj){ + yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4) + } + if(curPriceObj){ + if(curPrice!=weekendPriceObj.avg_price){ + def data = new BasicDBObject() + data.put("m_date", new Date()) + data.put("commid", dist._id.toString()) + data.put("mom", mom) + data.put("yoy", yoy) + data.put("avg_price", curPrice) + DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) + println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name + } + }else{ + def data = new BasicDBObject() + data.put("city", dist.city) + data.put("region", dist.region) + data.put("district_name", dist.name) + data.put("estate_type", dist.estate_type2) + data.put("date", curMonth) + data.put("avg_price", curPrice) + data.put("credit", weekendPriceObj.credit) + data.put("m_date", new Date()) + data.put("commid", dist._id.toString()) + data.put("mom", mom) + data.put("yoy", yoy) + DISTRICT_SELL_STAT.save(data) + println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name + } + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + } + + + //随机浮点数 + public static double nextDouble(final double min, final double max) throws Exception { + + DecimalFormat df = new DecimalFormat("#.00"); + if (max < min) { + throw new Exception("min < max"); + } + if (min == max) { + return min; + } + return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble()))); + } + + + def findPriceMap(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DISTRICT_STAT.find(q) + def priceMap = [:] + for(def d : dbcursor){ + priceMap.put(d.region +"##"+d.district_name, d) + } + return priceMap + } + + def findMonthlyPrice(city, monDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("date", monDate) + def dbcursor = DISTRICT_SELL_STAT.find(q) + def priceMap = [:] + for(def d : dbcursor){ + priceMap.put(d.region +"##"+d.district_name, d) + } + return priceMap + } + + + +// def updateDistrictSellStat(citys){ +// def index = 0 +// citys.each{city-> +// index ++ +// def stats = FZ_DISTRICT_STAT.find(new BasicDBObject("month",202209).append("city", city).append("avg_price", new BasicDBObject(QueryOperators.GTE, 0))).asList() +// if(stats){ +// batchUpdate(stats) +// } +// println "[${index}] "+city+" 完成" +// } +// } +// +// def batchUpdate(def stats){ +// def totalCount = new AtomicLong() +// final CountDownLatch latch = new CountDownLatch(stats.size()) +// +// for(def i=0;i -// fixedThreadPool.execute( -// new Runnable() { -// void run() { + fixedThreadPool.execute( + new Runnable() { + void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" if(dbo.name){ @@ -162,12 +162,12 @@ class DataStandardBatch extends BaseBatch{ }finally { latch.countDown() } -// } -// } -// ); + } + } + ); } -// latch.await() + latch.await() long end = System.currentTimeMillis() println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") @@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("city","北京") + qdata.put("city","福州") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index f954743e4..1caff80e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{ def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" - def esSources = ["中原地产":"中原地产"] + def esSources = [ "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] static main(args) { @@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = CommonUtils.findAllCity() + def citys = ["福州"]//CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 99355ab2a..22eedff7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202208); - def count2 = findTotalCount(source, 202209); + def count1 = findTotalCount(source, 202209); + def count2 = findTotalCount(source, 202210); println source +" "+count1 +" "+count2 } }