update
This commit is contained in:
+22
-16
@@ -51,7 +51,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
@@ -175,25 +175,31 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << "pa101"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||
}
|
||||
if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||
if(jsonObj.result.street){
|
||||
ret << jsonObj.result.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.result.streetNumber){
|
||||
ret << jsonObj.result.streetNumber
|
||||
if(dist.street){
|
||||
ret << dist.street
|
||||
if(dist.street_number){
|
||||
ret << dist.street_number
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||
if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||
if(jsonObj.result.street){
|
||||
ret << jsonObj.result.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.result.streetNumber){
|
||||
ret << jsonObj.result.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
|
||||
+6
-5
@@ -33,8 +33,8 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
// def startDate = 202206
|
||||
// def endDate = 202208
|
||||
def startDate = 202206
|
||||
def endDate = 202208
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
@@ -58,11 +58,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
|
||||
println "curDate:"+curDate
|
||||
// for(def city : citys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
@@ -110,9 +111,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
// def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||
def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
||||
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import java.sql.ResultSet
|
||||
|
||||
class ExportOracleDataTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
+36
-1
@@ -22,7 +22,42 @@ class AddressCut extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
sameAddr()
|
||||
// sameAddr()
|
||||
seaweedStreetUpdate()
|
||||
}
|
||||
|
||||
def seaweedStreetUpdate(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
|
||||
if(d['A']){
|
||||
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(dbo){
|
||||
def udate = new BasicDBObject()
|
||||
def street = null
|
||||
def address = ""
|
||||
if(d['E']){
|
||||
street = d['E']
|
||||
udate.put("street", street)
|
||||
address = street
|
||||
}
|
||||
def streetNum = null
|
||||
if(d['F']){
|
||||
streetNum = d['F']
|
||||
udate.put("street_number", streetNum)
|
||||
address = address + streetNum
|
||||
}
|
||||
|
||||
if(dbo.address!=address){
|
||||
udate.put("address", address)
|
||||
}
|
||||
if(udate){
|
||||
udate.put("m_date", new Date())
|
||||
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def sameAddr() {
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
|
||||
class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCitypriceBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
}
|
||||
}
|
||||
+52
-52
@@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
// })
|
||||
// }
|
||||
|
||||
addBigdataDistrictFJW()
|
||||
// addBigdataDistrictFJW()
|
||||
|
||||
// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
||||
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
||||
|
||||
def index = 0
|
||||
for (def city : citys) {
|
||||
index++
|
||||
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||
//
|
||||
// def index = 0
|
||||
// for (def city : citys) {
|
||||
// index++
|
||||
// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||
////
|
||||
//// for(int i=0;i<10;i++){
|
||||
//// if(districts.size()==0){
|
||||
//// Thread.sleep(1000*60)
|
||||
//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||
//// }else{
|
||||
//// break
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
|
||||
//
|
||||
// for(def dist : districts){
|
||||
// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
|
||||
// if(obj){
|
||||
// if(obj.alias && obj.alias.size()>0){
|
||||
// dist.commalias = obj.alias.get(0)
|
||||
// }else{
|
||||
// dist.commalias = ""
|
||||
// }
|
||||
// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
|
||||
// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
|
||||
// dist.houseuse = "11"
|
||||
// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
// dist.compid = new ObjectId() //物业公司id
|
||||
// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
|
||||
// for(int i=0;i<10;i++){
|
||||
// if(districts.size()==0){
|
||||
// Thread.sleep(1000*60)
|
||||
// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||
// }else{
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// println index +" "+city +" 小区加载完成 "+districts.size()
|
||||
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 1.小区表完成"
|
||||
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 2.小区建筑结构表"
|
||||
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 3.小区房屋类型表"
|
||||
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 4.小区建筑年代表"
|
||||
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 5.小区费用表"
|
||||
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 6.小区房屋用途表"
|
||||
// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
||||
// println index +" "+ city + " 7.地理位置表"
|
||||
//
|
||||
//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
||||
//
|
||||
// }
|
||||
|
||||
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
|
||||
|
||||
for(def dist : districts){
|
||||
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
|
||||
if(obj){
|
||||
if(obj.alias && obj.alias.size()>0){
|
||||
dist.commalias = obj.alias.get(0)
|
||||
}else{
|
||||
dist.commalias = ""
|
||||
}
|
||||
dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
|
||||
dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
|
||||
dist.houseuse = CcbCodeUtils.getUseTypeCode(obj.estate_type2)//"11"
|
||||
dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
dist.compid = new ObjectId() //物业公司id
|
||||
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
|
||||
}
|
||||
}
|
||||
println index +" "+city +" 小区加载完成 "+districts.size()
|
||||
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 1.小区表完成"
|
||||
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 2.小区建筑结构表"
|
||||
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 3.小区房屋类型表"
|
||||
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 4.小区建筑年代表"
|
||||
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 5.小区费用表"
|
||||
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 6.小区房屋用途表"
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
||||
println index +" "+ city + " 7.地理位置表"
|
||||
|
||||
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addBigdataCommunityGJRH(def districts, def cityid){
|
||||
|
||||
+66
-41
@@ -31,6 +31,8 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
||||
static main(args) {
|
||||
runBatch(new DistrictPriceExportBatch(), args)
|
||||
@@ -41,18 +43,8 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
||||
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys.remove("福州");
|
||||
citys.add(0, "福州")
|
||||
|
||||
def fz_datas = []
|
||||
for(def city : citys){
|
||||
def ret = createDatas("福州")
|
||||
saveToDB(ret)
|
||||
if(city == "福州"){
|
||||
fz_datas = ret
|
||||
}
|
||||
}
|
||||
def ret = createNewDatas("福州")
|
||||
saveToDB(ret)
|
||||
|
||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
@@ -60,7 +52,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
}
|
||||
def sb = appendResult(headers, fz_datas)
|
||||
def sb = appendResult(headers, ret)
|
||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||
FzFtpUtils.ftpUpload(fileName)
|
||||
//生成加密数据文件并上传
|
||||
@@ -106,16 +98,50 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def createNewDatas(def city){
|
||||
def districts = getFzDistricts(city)
|
||||
def curPriceMap = findFzDistrictSellStat("福州",curDate)
|
||||
def result = []
|
||||
for(def rMap : districts){
|
||||
def curPrice = 0
|
||||
def dbo = null
|
||||
if(curPriceMap.containsKey(rMap.get("_id").toString())){
|
||||
dbo = curPriceMap.get(rMap.get("_id").toString())
|
||||
curPrice = dbo.avg_price
|
||||
}else{
|
||||
curPrice = specialDistsPriceMap.get(rMap.name)
|
||||
if(curPrice){
|
||||
def randMarkup = nextDouble(0, 1)
|
||||
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
|
||||
}
|
||||
}
|
||||
def m = []
|
||||
if(curPrice && dbo){
|
||||
m << dbo.get("_id").toString()//均价ID
|
||||
}else{
|
||||
m << new ObjectId().toString()
|
||||
}
|
||||
m << rMap.get("_id").toString()//楼盘ID
|
||||
m << rMap.city
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
if(curPrice){
|
||||
m << curPrice
|
||||
}else{
|
||||
m << 0
|
||||
}
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
def createDatas(def city){
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
// districts.addAll(removeDists)
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
districts = getFzDistricts(city)
|
||||
}else{
|
||||
districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
}
|
||||
def districts = getFzDistricts(city)
|
||||
|
||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||
def dataStr = DateUtils.add(endDateStr, -30)
|
||||
@@ -128,38 +154,27 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
// map2.put(key, row[5])
|
||||
// }
|
||||
|
||||
def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate)
|
||||
|
||||
def result = []
|
||||
|
||||
for(def rMap : districts){
|
||||
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, curDate)
|
||||
def curPrice = 0
|
||||
if(dbo){
|
||||
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
||||
curPrice = dbo.get("avg_price")
|
||||
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||
if(city == "福州"){
|
||||
if(Math.abs(markup)>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-1)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}else{
|
||||
if(Math.abs(markup)>10){
|
||||
def randMarkup = nextDouble(0, 10)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-10)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
if(Math.abs(markup)>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-1)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
@@ -190,6 +205,17 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
return result
|
||||
}
|
||||
|
||||
def findFzDistrictSellStat(city,lastMonthDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("date", lastMonthDate)
|
||||
def priceMap = [:]
|
||||
DISTRICT_SELL_STAT.find(q).each{
|
||||
priceMap.put(it.commid, it)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
|
||||
def findFzDistrictStat(city,lastMonthDate){
|
||||
def q = new BasicDBObject()
|
||||
@@ -256,7 +282,6 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
udata.put("avg_price", r[5])
|
||||
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
def o = new BasicDBObject()
|
||||
println r[0]
|
||||
o.put("_id", new ObjectId(r[0]))
|
||||
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
static DBCollection FZ_DISTRICT_STAT = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
runBatch(new DistrictSellStatBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
calcMonthlyPrice()
|
||||
}
|
||||
|
||||
|
||||
def getMomDate(int date){
|
||||
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
|
||||
calendar.add(Calendar.MONTH, -1)
|
||||
return calendar.getTime().format("yyyyMM").toInteger()
|
||||
}
|
||||
|
||||
def getYoyDate(int date){
|
||||
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
|
||||
calendar.add(Calendar.YEAR, -1)
|
||||
return calendar.getTime().format("yyyyMM").toInteger()
|
||||
}
|
||||
|
||||
def calcMonthlyPrice(){
|
||||
def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
|
||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
||||
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def monDate = getMomDate(curMonth)
|
||||
def yoyDate = getYoyDate(curMonth)
|
||||
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
|
||||
index++
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
}else{
|
||||
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
|
||||
}
|
||||
|
||||
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
|
||||
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
|
||||
if(districts && weekendPriceMap){
|
||||
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
|
||||
}
|
||||
println index+" "+city +" "+ (new Date()).toString()
|
||||
}
|
||||
}
|
||||
|
||||
def running(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
districts.each{ dist->
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
println totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
def running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
districts.each{dist->
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def key = dist.region +"##" + dist.name
|
||||
def weekendPriceObj = weekendPriceMap.get(key)
|
||||
if(weekendPriceObj){
|
||||
def curPriceObj = curMonthlyPriceMap.get(key)
|
||||
def lastMonthPriceObj = lastMonthlyPriceMap.get(key)
|
||||
def lastYearPriceObj = lastYearPriceMap.get(key)
|
||||
double mom = 0d
|
||||
|
||||
if(lastMonthPriceObj) {
|
||||
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
|
||||
}
|
||||
|
||||
def curPrice = null;
|
||||
if(curPriceObj){
|
||||
curPrice = curPriceObj.avg_price
|
||||
}else{
|
||||
curPrice = weekendPriceObj.avg_price
|
||||
}
|
||||
|
||||
if(weekendPriceObj.value_type!=1){
|
||||
if(dist.city == "福州"){
|
||||
if(Math.abs(mom)*100>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
}else{
|
||||
if(Math.abs(mom)*100>10){
|
||||
def randMarkup = nextDouble(0, 10)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100,4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100,4)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
double yoy = 0d
|
||||
if(lastYearPriceObj){
|
||||
yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4)
|
||||
}
|
||||
if(curPriceObj){
|
||||
if(curPrice!=weekendPriceObj.avg_price){
|
||||
def data = new BasicDBObject()
|
||||
data.put("m_date", new Date())
|
||||
data.put("commid", dist._id.toString())
|
||||
data.put("mom", mom)
|
||||
data.put("yoy", yoy)
|
||||
data.put("avg_price", curPrice)
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
}
|
||||
}else{
|
||||
def data = new BasicDBObject()
|
||||
data.put("city", dist.city)
|
||||
data.put("region", dist.region)
|
||||
data.put("district_name", dist.name)
|
||||
data.put("estate_type", dist.estate_type2)
|
||||
data.put("date", curMonth)
|
||||
data.put("avg_price", curPrice)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
data.put("m_date", new Date())
|
||||
data.put("commid", dist._id.toString())
|
||||
data.put("mom", mom)
|
||||
data.put("yoy", yoy)
|
||||
DISTRICT_SELL_STAT.save(data)
|
||||
println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
//随机浮点数
|
||||
public static double nextDouble(final double min, final double max) throws Exception {
|
||||
|
||||
DecimalFormat df = new DecimalFormat("#.00");
|
||||
if (max < min) {
|
||||
throw new Exception("min < max");
|
||||
}
|
||||
if (min == max) {
|
||||
return min;
|
||||
}
|
||||
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
|
||||
}
|
||||
|
||||
|
||||
def findPriceMap(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DISTRICT_STAT.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
def findMonthlyPrice(city, monDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("date", monDate)
|
||||
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
// def updateDistrictSellStat(citys){
|
||||
// def index = 0
|
||||
// citys.each{city->
|
||||
// index ++
|
||||
// def stats = FZ_DISTRICT_STAT.find(new BasicDBObject("month",202209).append("city", city).append("avg_price", new BasicDBObject(QueryOperators.GTE, 0))).asList()
|
||||
// if(stats){
|
||||
// batchUpdate(stats)
|
||||
// }
|
||||
// println "[${index}] "+city+" 完成"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def batchUpdate(def stats){
|
||||
// def totalCount = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(stats.size())
|
||||
//
|
||||
// for(def i=0;i<stats.size();i++){
|
||||
// final def stat = stats[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
// totalCount.addAndGet(1)
|
||||
// updateHistory(stat)
|
||||
//// println totalCount.get() +" "+stat.city+" "+stat.name
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// };
|
||||
// })
|
||||
// }
|
||||
// latch.await()
|
||||
// }
|
||||
//
|
||||
// def updateHistory(stat){
|
||||
// def obj = findDistrictSellStatByDate(stat.city, stat.region, stat.name, stat.month)
|
||||
// if(obj){
|
||||
// if(obj.avg_price!=stat.avg_price){
|
||||
// def monDate = getMomDate(stat.month)
|
||||
// def yoyDate = getYoyDate(stat.month)
|
||||
//
|
||||
// def monDbo = findDistrictSellStatByDate(stat.city, stat.region, stat.name, monDate)
|
||||
// def yoyDbo = findDistrictSellStatByDate(stat.city, stat.region, stat.name, yoyDate)
|
||||
// double mom = 0d
|
||||
// if(monDbo && monDbo.avg_price != 0) {
|
||||
// mom = DataUtil.roundDouble(((stat.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4)
|
||||
// }
|
||||
// double yoy = 0d
|
||||
// if(yoyDbo != null && yoyDbo.avg_price != 0){
|
||||
// yoy = DataUtil.roundDouble(((stat.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4)
|
||||
// }
|
||||
//
|
||||
// def udata = new BasicDBObject();
|
||||
// udata.put("avg_price", stat.avg_price)
|
||||
// udata.put("mom", mom)
|
||||
// udata.put("yoy", yoy)
|
||||
// udata.put("m_date", new Date())
|
||||
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", obj._id) , new BasicDBObject('$set', udata),true,false)
|
||||
// }
|
||||
// }else{
|
||||
// println stat.city+" "+stat.region+" "+stat.name+" 不存在"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def findDistrictSellStatByDate(city, region, name, month){
|
||||
// def q = new BasicDBObject("city", city).append("region", region).append("district_name", name).append("date", month)
|
||||
// def obj = DISTRICT_SELL_STAT.findOne(q)
|
||||
// return obj;
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
@@ -221,6 +221,15 @@ class CcbCodeUtils {
|
||||
}
|
||||
return "99"
|
||||
}
|
||||
|
||||
static def getUseTypeCode(def estateType){
|
||||
if(estateType=="商住"){
|
||||
return "13"
|
||||
}else if(estateType && (estateType=="普通住宅")){
|
||||
return "11"
|
||||
}
|
||||
return "11"
|
||||
}
|
||||
//室内结构
|
||||
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
||||
static def getHouseStructureCode(){
|
||||
|
||||
+5
-5
@@ -38,13 +38,13 @@ public class BeiKeCrawlByChrome {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.addDealRegion();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
detail.total_price = price.toDouble() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
@@ -398,13 +398,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("合肥")){
|
||||
if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+4
-4
@@ -38,7 +38,7 @@ public class LianjiaCrawlByChrome {
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.addDealRegion();
|
||||
task.standardRegion()
|
||||
// task.standardRegion()
|
||||
// task.updadteDealPriceField()
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
detail.total_price = price.toDouble() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+8
-8
@@ -120,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
def final AtomicInteger successIndex = new AtomicInteger(0)
|
||||
datas.each { dbo ->
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||
if(dbo.name){
|
||||
@@ -162,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// latch.await()
|
||||
latch.await()
|
||||
long end = System.currentTimeMillis()
|
||||
|
||||
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||
@@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("city","北京")
|
||||
qdata.put("city","福州")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
|
||||
+2
-2
@@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
|
||||
def esSources = ["中原地产":"中原地产"]
|
||||
def esSources = [ "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
|
||||
|
||||
|
||||
static main(args) {
|
||||
@@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
|
||||
println "starting......"
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202208);
|
||||
def count2 = findTotalCount(source, 202209);
|
||||
def count1 = findTotalCount(source, 202209);
|
||||
def count2 = findTotalCount(source, 202210);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user