diff --git a/projects/fangjia-coresite/pom.xml b/projects/fangjia-coresite/pom.xml
index 05fa7f275..72cefc4b9 100644
--- a/projects/fangjia-coresite/pom.xml
+++ b/projects/fangjia-coresite/pom.xml
@@ -242,5 +242,13 @@
1.0.0
jdk18
+
+
+
+
+
+
+
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy
index fafe61ba9..2439e06d8 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy
@@ -51,7 +51,7 @@ class BigdataCommunityBatch extends BaseBatch{
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
- def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
+ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
@@ -175,25 +175,31 @@ class BigdataCommunityBatch extends BaseBatch{
ret << "pa101"
ret << ""
if(dist.address){
- def jsonObj = null
- if(dist.address){
- jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
- }
- if(jsonObj && jsonObj.result && jsonObj.result.street){
- if(jsonObj.result.street){
- ret << jsonObj.result.street
- }else{
- ret << ""
- }
-
- if(jsonObj.result.streetNumber){
- ret << jsonObj.result.streetNumber
+ if(dist.street){
+ ret << dist.street
+ if(dist.street_number){
+ ret << dist.street_number
}else{
ret << ""
}
}else{
- ret << ""
- ret << ""
+ def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
+ if(jsonObj && jsonObj.result && jsonObj.result.street){
+ if(jsonObj.result.street){
+ ret << jsonObj.result.street
+ }else{
+ ret << ""
+ }
+
+ if(jsonObj.result.streetNumber){
+ ret << jsonObj.result.streetNumber
+ }else{
+ ret << ""
+ }
+ }else{
+ ret << ""
+ ret << ""
+ }
}
}else{
ret << ""
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
index 789603d29..a1a81702d 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
@@ -33,8 +33,8 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
-// def startDate = 202206
-// def endDate = 202208
+ def startDate = 202206
+ def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -58,11 +58,12 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
- def citys = ["ÉϺ£"]//CcbCommonUtils.getRunningCitys(args)
+ def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
+ println "curDate:"+curDate
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
@@ -110,9 +111,9 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
-// def lianjiaPriceMap = findThirdPartyPrice("ÉϺ£")
+ def lianjiaPriceMap = findThirdPartyPrice("ÉϺ£")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
- def lianjiaPriceMap = getDistrictStatByCity("ÉϺ£", startDate)
+// def lianjiaPriceMap = getDistrictStatByCity("ÉϺ£", startDate)
for(def i=0;i
+ if(d['A']){
+ def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
+ if(dbo){
+ def udate = new BasicDBObject()
+ def street = null
+ def address = ""
+ if(d['E']){
+ street = d['E']
+ udate.put("street", street)
+ address = street
+ }
+ def streetNum = null
+ if(d['F']){
+ streetNum = d['F']
+ udate.put("street_number", streetNum)
+ address = address + streetNum
+ }
+
+ if(dbo.address!=address){
+ udate.put("address", address)
+ }
+ if(udate){
+ udate.put("m_date", new Date())
+ DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
+ }
+
+ }
+ }
+
+ })
+=======
// sameAddr() //ÏàͬµØÖ·Ð¡Çø
// roadMatch()
addAnjukeRoad()
@@ -93,6 +131,7 @@ class AddressCut extends BaseBatch{
})
+>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
}
def sameAddr() {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy
new file mode 100644
index 000000000..1656648bd
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy
@@ -0,0 +1,15 @@
+package com.fangjia.script.ccbevaluation.ronghe
+
+import com.fangjia.script.BaseBatch
+
+class BigdataCitypriceBatch extends BaseBatch{
+
+ static main(args) {
+ runBatch(new BigdataCitypriceBatch(), args)
+ }
+
+ @Override
+ void process(Object args) {
+
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy
index b8116db22..77d788a46 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy
@@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// })
// }
- addBigdataDistrictFJW()
+// addBigdataDistrictFJW()
-// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
+ def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
+
+ def index = 0
+ for (def city : citys) {
+ index++
+ def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
//
-// def index = 0
-// for (def city : citys) {
-// index++
-// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
-////
-//// for(int i=0;i<10;i++){
-//// if(districts.size()==0){
-//// Thread.sleep(1000*60)
-//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
-//// }else{
-//// break
-//// }
-//// }
-//
-// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
-//
-// for(def dist : districts){
-// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
-// if(obj){
-// if(obj.alias && obj.alias.size()>0){
-// dist.commalias = obj.alias.get(0)
-// }else{
-// dist.commalias = ""
-// }
-// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
-// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
-// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
-// dist.houseuse = "11"
-// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
-// dist.compid = new ObjectId() //ÎïÒµ¹«Ë¾id
-// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
+// for(int i=0;i<10;i++){
+// if(districts.size()==0){
+// Thread.sleep(1000*60)
+// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
+// }else{
+// break
// }
// }
-// println index +" "+city +" Ð¡Çø¼ÓÔØÍê³É "+districts.size()
-// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
-// println index +" "+city +" 1.Ð¡Çø±íÍê³É"
-// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
-// println index +" "+city +" 2.Ð¡Çø½¨Öþ½á¹¹±í"
-// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
-// println index +" "+city +" 3.Ð¡Çø·¿ÎÝÀàÐͱí"
-// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
-// println index +" "+city +" 4.Ð¡Çø½¨ÖþÄê´ú±í"
-// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
-// println index +" "+city +" 5.Ð¡Çø·ÑÓñí"
-// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
-// println index +" "+city +" 6.Ð¡Çø·¿ÎÝÓÃ;±í"
-// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
-// println index +" "+ city + " 7.µØÀíλÖñí"
-//
-//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//Ð޸ijɽ»¼Ç¼housestyle Ò»¶¨ÒªÔËÐÐ
-//
-// }
+
+ def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
+
+ for(def dist : districts){
+ def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
+ if(obj){
+ if(obj.alias && obj.alias.size()>0){
+ dist.commalias = obj.alias.get(0)
+ }else{
+ dist.commalias = ""
+ }
+ dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
+ dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
+ dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
+ dist.houseuse = CcbCodeUtils.getUseTypeCode(obj.estate_type2)//"11"
+ dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
+ dist.compid = new ObjectId() //ÎïÒµ¹«Ë¾id
+ dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
+ }
+ }
+ println index +" "+city +" Ð¡Çø¼ÓÔØÍê³É "+districts.size()
+ addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
+ println index +" "+city +" 1.Ð¡Çø±íÍê³É"
+ addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
+ println index +" "+city +" 2.Ð¡Çø½¨Öþ½á¹¹±í"
+ addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
+ println index +" "+city +" 3.Ð¡Çø·¿ÎÝÀàÐͱí"
+ addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
+ println index +" "+city +" 4.Ð¡Çø½¨ÖþÄê´ú±í"
+ addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
+ println index +" "+city +" 5.Ð¡Çø·ÑÓñí"
+ addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
+ println index +" "+city +" 6.Ð¡Çø·¿ÎÝÓÃ;±í"
+ addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
+ println index +" "+ city + " 7.µØÀíλÖñí"
+
+// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//Ð޸ijɽ»¼Ç¼housestyle Ò»¶¨ÒªÔËÐÐ
+
+ }
}
def addBigdataCommunityGJRH(def districts, def cityid){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy
index 517bfef5c..8d77112a6 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy
@@ -31,6 +31,8 @@ class DistrictPriceExportBatch extends BaseBatch{
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
+ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
+
def specialDistsPriceMap = ["½ò̩дå7ºÅ¡¢9-17ºÅ":53620,"½ò̩дå14ºÅ":76230,"½ò̩дå1-6ºÅ¡¢8ºÅ":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
@@ -41,18 +43,8 @@ class DistrictPriceExportBatch extends BaseBatch{
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
- def citys = CommonDatautils.findAllCity()
- citys.remove("¸£ÖÝ");
- citys.add(0, "¸£ÖÝ")
-
- def fz_datas = []
- for(def city : citys){
- def ret = createDatas("¸£ÖÝ")
- saveToDB(ret)
- if(city == "¸£ÖÝ"){
- fz_datas = ret
- }
- }
+ def ret = createNewDatas("¸£ÖÝ")
+ saveToDB(ret)
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
@@ -60,7 +52,7 @@ class DistrictPriceExportBatch extends BaseBatch{
fileName = "D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
- def sb = appendResult(headers, fz_datas)
+ def sb = appendResult(headers, ret)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//Éú³É¼ÓÃÜÊý¾ÝÎļþ²¢ÉÏ´«
@@ -106,16 +98,50 @@ class DistrictPriceExportBatch extends BaseBatch{
return rets
}
+
+ def createNewDatas(def city){
+ def districts = getFzDistricts(city)
+ def curPriceMap = findFzDistrictSellStat("¸£ÖÝ",curDate)
+ def result = []
+ for(def rMap : districts){
+ def curPrice = 0
+ def dbo = null
+ if(curPriceMap.containsKey(rMap.get("_id").toString())){
+ dbo = curPriceMap.get(rMap.get("_id").toString())
+ curPrice = dbo.avg_price
+ }else{
+ curPrice = specialDistsPriceMap.get(rMap.name)
+ if(curPrice){
+ def randMarkup = nextDouble(0, 1)
+ curPrice = (int)(curPrice*randMarkup/100 + curPrice)
+ }
+ }
+ def m = []
+ if(curPrice && dbo){
+ m << dbo.get("_id").toString()//¾ù¼ÛID
+ }else{
+ m << new ObjectId().toString()
+ }
+ m << rMap.get("_id").toString()//Â¥ÅÌID
+ m << rMap.city
+ m << rMap.region
+ m << rMap.name
+ if(curPrice){
+ m << curPrice
+ }else{
+ m << 0
+ }
+ m << curDate.toInteger();
+ result << m
+ }
+ return result
+ }
+
def createDatas(def city){
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// districts.addAll(removeDists)
- def districts = []
- if(city == "¸£ÖÝ"){
- districts = getFzDistricts(city)
- }else{
- districts = CommonDatautils.findAllDistricts(city, "district", null);
- }
+ def districts = getFzDistricts(city)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
@@ -128,38 +154,27 @@ class DistrictPriceExportBatch extends BaseBatch{
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
// }
-
def lastMonthPriceMap = findFzDistrictStat("¸£ÖÝ",lastMonthDate)
+
def result = []
+
for(def rMap : districts){
- def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
+ def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, curDate)
def curPrice = 0
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice && dbo.value_type!=1){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
- if(city == "¸£ÖÝ"){
- if(Math.abs(markup)>3){
- def randMarkup = nextDouble(0, 3)
- if(markup < 0){
- markup = randMarkup*(-1)
- }else{
- markup = randMarkup
- }
- curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
- }
- }else{
- if(Math.abs(markup)>10){
- def randMarkup = nextDouble(0, 10)
- if(markup < 0){
- markup = randMarkup*(-10)
- }else{
- markup = randMarkup
- }
- curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
+ if(Math.abs(markup)>3){
+ def randMarkup = nextDouble(0, 3)
+ if(markup < 0){
+ markup = randMarkup*(-1)
+ }else{
+ markup = randMarkup
}
+ curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
}else{
@@ -190,6 +205,17 @@ class DistrictPriceExportBatch extends BaseBatch{
return result
}
+ def findFzDistrictSellStat(city,lastMonthDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("date", lastMonthDate)
+ def priceMap = [:]
+ DISTRICT_SELL_STAT.find(q).each{
+ priceMap.put(it.commid, it)
+ }
+ return priceMap
+ }
+
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
@@ -256,7 +282,6 @@ class DistrictPriceExportBatch extends BaseBatch{
udata.put("avg_price", r[5])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
- def udata = new BasicDBObject()
def o = new BasicDBObject()
println r[0]
o.put("_id", new ObjectId(r[0]))
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy
new file mode 100644
index 000000000..a11d0ce17
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy
@@ -0,0 +1,321 @@
+package com.fangjia.script.fuzhou
+
+import com.fangjia.core.util.DataUtil
+import com.fangjia.script.BaseBatch
+import com.fangjia.script.BatchOwner
+import com.fangjia.script.Watcher
+import com.fangjia.script.util.CommonUtils
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.QueryOperators
+
+import java.text.DecimalFormat
+import java.util.concurrent.CopyOnWriteArrayList
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+
+@BatchOwner(Watcher.GaoJuan)
+class DistrictSellStatBatch extends BaseBatch {
+
+ static DBCollection FZ_DISTRICT_STAT = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
+
+ static DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
+ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
+ static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
+
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
+
+
+ public static void main(String[] args) {
+ runBatch(new DistrictSellStatBatch(), args);
+ }
+
+ @Override
+ void process(Object args) {
+ calcMonthlyPrice()
+ }
+
+
+ def getMomDate(int date){
+
+ Calendar calendar = Calendar.getInstance()
+ calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
+ calendar.add(Calendar.MONTH, -1)
+ return calendar.getTime().format("yyyyMM").toInteger()
+ }
+
+ def getYoyDate(int date){
+
+ Calendar calendar = Calendar.getInstance()
+ calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
+ calendar.add(Calendar.YEAR, -1)
+ return calendar.getTime().format("yyyyMM").toInteger()
+ }
+
+ def calcMonthlyPrice(){
+ def citys =["°°É½", '´óÇì', "ºâÑô", "ÄÏÑô", "ÆÎÌï"] // CommonUtils.findAllCity()
+ def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
+
+ def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
+
+ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
+ def monDate = getMomDate(curMonth)
+ def yoyDate = getYoyDate(curMonth)
+
+ def index = 0
+ for(def city : citys){
+
+ index++
+ def districts = []
+ if(city == "¸£ÖÝ"){
+ districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
+ }else{
+ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
+ }
+
+ def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//ÉϸöÔÂÐ¡Çø¾ù¼Û
+ def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//µ±ÔÂÐ¡ÇøÔ¾ù¼Û
+ def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//ÉϸöÔÂÐ¡ÇøÔ¾ù¼Û
+ def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//È¥ÄêÐ¡ÇøÔ¾ù¼Û
+ if(districts && weekendPriceMap){
+ running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
+ }
+ println index+" "+city +" "+ (new Date()).toString()
+ }
+ }
+
+ def running(districts){
+ def totalCount = new AtomicLong()
+ final CountDownLatch latch = new CountDownLatch(districts.size())
+ districts.each{ dist->
+ fixedThreadPool.execute(new Runnable() {
+ void run() {
+ try {
+ totalCount.addAndGet(1)
+ println totalCount.get()+" "+ dist.city+" "+dist.name
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ }
+ })
+ }
+ }
+
+ def running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap){
+ def totalCount = new AtomicLong()
+ final CountDownLatch latch = new CountDownLatch(districts.size())
+ def rets = new CopyOnWriteArrayList()
+
+ districts.each{dist->
+ fixedThreadPool.execute(new Runnable() {
+ void run() {
+ try {
+ totalCount.addAndGet(1)
+ def key = dist.region +"##" + dist.name
+ def weekendPriceObj = weekendPriceMap.get(key)
+ if(weekendPriceObj){
+ def curPriceObj = curMonthlyPriceMap.get(key)
+ def lastMonthPriceObj = lastMonthlyPriceMap.get(key)
+ def lastYearPriceObj = lastYearPriceMap.get(key)
+ double mom = 0d
+
+ if(lastMonthPriceObj) {
+ mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
+ }
+
+ def curPrice = null;
+ if(curPriceObj){
+ curPrice = curPriceObj.avg_price
+ }else{
+ curPrice = weekendPriceObj.avg_price
+ }
+
+ if(weekendPriceObj.value_type!=1){
+ if(dist.city == "¸£ÖÝ"){
+ if(Math.abs(mom)*100>3){
+ def randMarkup = nextDouble(0, 3)
+ if(mom < 0){
+ mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
+ }else{
+ mom = DataUtil.roundDouble(randMarkup/100, 4)
+ }
+ curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
+ }
+ }else{
+ if(Math.abs(mom)*100>10){
+ def randMarkup = nextDouble(0, 10)
+ if(mom < 0){
+ mom = DataUtil.roundDouble(randMarkup*(-1)/100,4)
+ }else{
+ mom = DataUtil.roundDouble(randMarkup/100,4)
+ }
+ curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
+ }
+ }
+ }
+
+ double yoy = 0d
+ if(lastYearPriceObj){
+ yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4)
+ }
+ if(curPriceObj){
+ if(curPrice!=weekendPriceObj.avg_price){
+ def data = new BasicDBObject()
+ data.put("m_date", new Date())
+ data.put("commid", dist._id.toString())
+ data.put("mom", mom)
+ data.put("yoy", yoy)
+ data.put("avg_price", curPrice)
+ DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
+ println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
+ }
+ }else{
+ def data = new BasicDBObject()
+ data.put("city", dist.city)
+ data.put("region", dist.region)
+ data.put("district_name", dist.name)
+ data.put("estate_type", dist.estate_type2)
+ data.put("date", curMonth)
+ data.put("avg_price", curPrice)
+ data.put("credit", weekendPriceObj.credit)
+ data.put("m_date", new Date())
+ data.put("commid", dist._id.toString())
+ data.put("mom", mom)
+ data.put("yoy", yoy)
+ DISTRICT_SELL_STAT.save(data)
+ println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name
+ }
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ };
+ })
+ }
+ latch.await()
+ }
+
+
+ //Ëæ»ú¸¡µãÊý
+ public static double nextDouble(final double min, final double max) throws Exception {
+
+ DecimalFormat df = new DecimalFormat("#.00");
+ if (max < min) {
+ throw new Exception("min < max");
+ }
+ if (min == max) {
+ return min;
+ }
+ return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
+ }
+
+
+ def findPriceMap(city, weekend){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("weekend", weekend)
+ def dbcursor = DISTRICT_STAT.find(q)
+ def priceMap = [:]
+ for(def d : dbcursor){
+ priceMap.put(d.region +"##"+d.district_name, d)
+ }
+ return priceMap
+ }
+
+ def findMonthlyPrice(city, monDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("date", monDate)
+ def dbcursor = DISTRICT_SELL_STAT.find(q)
+ def priceMap = [:]
+ for(def d : dbcursor){
+ priceMap.put(d.region +"##"+d.district_name, d)
+ }
+ return priceMap
+ }
+
+
+
+// def updateDistrictSellStat(citys){
+// def index = 0
+// citys.each{city->
+// index ++
+// def stats = FZ_DISTRICT_STAT.find(new BasicDBObject("month",202209).append("city", city).append("avg_price", new BasicDBObject(QueryOperators.GTE, 0))).asList()
+// if(stats){
+// batchUpdate(stats)
+// }
+// println "[${index}] "+city+" Íê³É"
+// }
+// }
+//
+// def batchUpdate(def stats){
+// def totalCount = new AtomicLong()
+// final CountDownLatch latch = new CountDownLatch(stats.size())
+//
+// for(def i=0;i
-// fixedThreadPool.execute(
-// new Runnable() {
-// void run() {
+ fixedThreadPool.execute(
+ new Runnable() {
+ void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})¡¾${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}¡¿"
if(dbo.name){
@@ -162,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
}finally {
latch.countDown()
}
-// }
-// }
-// );
+ }
+ }
+ );
}
-// latch.await()
+ latch.await()
long end = System.currentTimeMillis()
println "µÚ¡¾${rowIndex}¡¿´ÎÖÐÆ¥Åä³É¹¦µÄ£º"+successCount+" ¸ö£¬ºÄʱ£º"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
@@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
-// qdata.put("city","±±¾©")
+ qdata.put("city","¸£ÖÝ")
// qdata.put("source","°²¾Ó¿Í")
// qdata.put("name",Pattern.compile("¹¨¼ÒÕ¬.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
index f954743e4..1caff80e1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
@@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{
def sourceMap = ["ke.com":"±´¿ÇÕÒ·¿","cityhouse.cn":"³ÇÊз¿²ú"]
// "¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐԵزú":"ÖÐԵزú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú"
- def esSources = ["ÖÐԵزú":"ÖÐԵزú"]
+ def esSources = [ "¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐԵزú":"ÖÐԵزú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú"]
static main(args) {
@@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
- def citys = CommonUtils.findAllCity()
+ def citys = ["¸£ÖÝ"]//CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
index 99355ab2a..22eedff7f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐԵزú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐԵزú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"]
for(def source :sources){
- def count1 = findTotalCount(source, 202208);
- def count2 = findTotalCount(source, 202209);
+ def count1 = findTotalCount(source, 202209);
+ def count2 = findTotalCount(source, 202210);
println source +" "+count1 +" "+count2
}
}