Merge branch 'script_gj' into 'master'

Script gj

See merge request !431
This commit is contained in:
2022-11-09 13:45:24 +08:00
15 changed files with 560 additions and 136 deletions
+8
View File
@@ -242,5 +242,13 @@
<version>1.0.0</version>
<classifier>jdk18</classifier>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-devtools</artifactId>-->
<!-- &lt;!&ndash; optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 &ndash;&gt;-->
<!-- <optional>true</optional>-->
<!-- </dependency>-->
</dependencies>
</project>
@@ -51,7 +51,7 @@ class BigdataCommunityBatch extends BaseBatch{
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
@@ -175,25 +175,31 @@ class BigdataCommunityBatch extends BaseBatch{
ret << "pa101"
ret << ""
if(dist.address){
def jsonObj = null
if(dist.address){
jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
}
if(jsonObj && jsonObj.result && jsonObj.result.street){
if(jsonObj.result.street){
ret << jsonObj.result.street
}else{
ret << ""
}
if(jsonObj.result.streetNumber){
ret << jsonObj.result.streetNumber
if(dist.street){
ret << dist.street
if(dist.street_number){
ret << dist.street_number
}else{
ret << ""
}
}else{
ret << ""
ret << ""
def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
if(jsonObj && jsonObj.result && jsonObj.result.street){
if(jsonObj.result.street){
ret << jsonObj.result.street
}else{
ret << ""
}
if(jsonObj.result.streetNumber){
ret << jsonObj.result.streetNumber
}else{
ret << ""
}
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
@@ -33,8 +33,8 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
// def startDate = 202206
// def endDate = 202208
def startDate = 202206
def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -58,11 +58,12 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
@@ -110,9 +111,9 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
def lianjiaPriceMap = findThirdPartyPrice("上海")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
@@ -14,7 +14,7 @@ import java.sql.ResultSet
class ExportOracleDataTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
@@ -39,6 +39,44 @@ class AddressCut extends BaseBatch{
@Override
public void process(Object args) {
<<<<<<< HEAD
// sameAddr()
seaweedStreetUpdate()
}
def seaweedStreetUpdate(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
if(d['A']){
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dbo){
def udate = new BasicDBObject()
def street = null
def address = ""
if(d['E']){
street = d['E']
udate.put("street", street)
address = street
}
def streetNum = null
if(d['F']){
streetNum = d['F']
udate.put("street_number", streetNum)
address = address + streetNum
}
if(dbo.address!=address){
udate.put("address", address)
}
if(udate){
udate.put("m_date", new Date())
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
}
}
}
})
=======
// sameAddr() //相同地址小区
// roadMatch()
addAnjukeRoad()
@@ -93,6 +131,7 @@ class AddressCut extends BaseBatch{
})
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
}
def sameAddr() {
@@ -0,0 +1,15 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.BaseBatch
class BigdataCitypriceBatch extends BaseBatch{
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
}
@Override
void process(Object args) {
}
}
@@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// })
// }
addBigdataDistrictFJW()
// addBigdataDistrictFJW()
// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
def index = 0
for (def city : citys) {
index++
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
//
// def index = 0
// for (def city : citys) {
// index++
// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
////
//// for(int i=0;i<10;i++){
//// if(districts.size()==0){
//// Thread.sleep(1000*60)
//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
//// }else{
//// break
//// }
//// }
//
// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
//
// for(def dist : districts){
// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
// if(obj){
// if(obj.alias && obj.alias.size()>0){
// dist.commalias = obj.alias.get(0)
// }else{
// dist.commalias = ""
// }
// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
// dist.houseuse = "11"
// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
// dist.compid = new ObjectId() //物业公司id
// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
// for(int i=0;i<10;i++){
// if(districts.size()==0){
// Thread.sleep(1000*60)
// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
// }else{
// break
// }
// }
// println index +" "+city +" 小区加载完成 "+districts.size()
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 1.小区表完成"
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 2.小区建筑结构表"
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 3.小区房屋类型表"
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 4.小区建筑年代表"
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 5.小区费用表"
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 6.小区房屋用途表"
// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
// println index +" "+ city + " 7.地理位置表"
//
//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
//
// }
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
for(def dist : districts){
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
if(obj){
if(obj.alias && obj.alias.size()>0){
dist.commalias = obj.alias.get(0)
}else{
dist.commalias = ""
}
dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
dist.houseuse = CcbCodeUtils.getUseTypeCode(obj.estate_type2)//"11"
dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
dist.compid = new ObjectId() //物业公司id
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
}
}
println index +" "+city +" 小区加载完成 "+districts.size()
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
println index +" "+city +" 1.小区表完成"
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 2.小区建筑结构表"
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 3.小区房屋类型表"
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 4.小区建筑年代表"
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 5.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
println index +" "+ city + " 7.地理位置表"
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
}
}
def addBigdataCommunityGJRH(def districts, def cityid){
@@ -31,6 +31,8 @@ class DistrictPriceExportBatch extends BaseBatch{
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
@@ -41,18 +43,8 @@ class DistrictPriceExportBatch extends BaseBatch{
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
def citys = CommonDatautils.findAllCity()
citys.remove("福州");
citys.add(0, "福州")
def fz_datas = []
for(def city : citys){
def ret = createDatas("福州")
saveToDB(ret)
if(city == "福州"){
fz_datas = ret
}
}
def ret = createNewDatas("福州")
saveToDB(ret)
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
@@ -60,7 +52,7 @@ class DistrictPriceExportBatch extends BaseBatch{
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
def sb = appendResult(headers, fz_datas)
def sb = appendResult(headers, ret)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//生成加密数据文件并上传
@@ -106,16 +98,50 @@ class DistrictPriceExportBatch extends BaseBatch{
return rets
}
def createNewDatas(def city){
def districts = getFzDistricts(city)
def curPriceMap = findFzDistrictSellStat("福州",curDate)
def result = []
for(def rMap : districts){
def curPrice = 0
def dbo = null
if(curPriceMap.containsKey(rMap.get("_id").toString())){
dbo = curPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.avg_price
}else{
curPrice = specialDistsPriceMap.get(rMap.name)
if(curPrice){
def randMarkup = nextDouble(0, 1)
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
}
}
def m = []
if(curPrice && dbo){
m << dbo.get("_id").toString()//均价ID
}else{
m << new ObjectId().toString()
}
m << rMap.get("_id").toString()//楼盘ID
m << rMap.city
m << rMap.region
m << rMap.name
if(curPrice){
m << curPrice
}else{
m << 0
}
m << curDate.toInteger();
result << m
}
return result
}
def createDatas(def city){
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// districts.addAll(removeDists)
def districts = []
if(city == "福州"){
districts = getFzDistricts(city)
}else{
districts = CommonDatautils.findAllDistricts(city, "district", null);
}
def districts = getFzDistricts(city)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
@@ -128,38 +154,27 @@ class DistrictPriceExportBatch extends BaseBatch{
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
// }
def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate)
def result = []
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, curDate)
def curPrice = 0
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice && dbo.value_type!=1){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(city == "福州"){
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}else{
if(Math.abs(markup)>10){
def randMarkup = nextDouble(0, 10)
if(markup < 0){
markup = randMarkup*(-10)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
}else{
@@ -190,6 +205,17 @@ class DistrictPriceExportBatch extends BaseBatch{
return result
}
def findFzDistrictSellStat(city,lastMonthDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("date", lastMonthDate)
def priceMap = [:]
DISTRICT_SELL_STAT.find(q).each{
priceMap.put(it.commid, it)
}
return priceMap
}
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
@@ -256,7 +282,6 @@ class DistrictPriceExportBatch extends BaseBatch{
udata.put("avg_price", r[5])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
def o = new BasicDBObject()
println r[0]
o.put("_id", new ObjectId(r[0]))
@@ -0,0 +1,321 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.text.DecimalFormat
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
@BatchOwner(Watcher.GaoJuan)
class DistrictSellStatBatch extends BaseBatch {
static DBCollection FZ_DISTRICT_STAT = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static void main(String[] args) {
runBatch(new DistrictSellStatBatch(), args);
}
@Override
void process(Object args) {
calcMonthlyPrice()
}
def getMomDate(int date){
Calendar calendar = Calendar.getInstance()
calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
calendar.add(Calendar.MONTH, -1)
return calendar.getTime().format("yyyyMM").toInteger()
}
def getYoyDate(int date){
Calendar calendar = Calendar.getInstance()
calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
calendar.add(Calendar.YEAR, -1)
return calendar.getTime().format("yyyyMM").toInteger()
}
def calcMonthlyPrice(){
def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def monDate = getMomDate(curMonth)
def yoyDate = getYoyDate(curMonth)
def index = 0
for(def city : citys){
index++
def districts = []
if(city == "福州"){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
}
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
if(districts && weekendPriceMap){
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
}
println index+" "+city +" "+ (new Date()).toString()
}
}
def running(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
districts.each{ dist->
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
println totalCount.get()+" "+ dist.city+" "+dist.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
}
def running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
districts.each{dist->
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name
def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){
def curPriceObj = curMonthlyPriceMap.get(key)
def lastMonthPriceObj = lastMonthlyPriceMap.get(key)
def lastYearPriceObj = lastYearPriceMap.get(key)
double mom = 0d
if(lastMonthPriceObj) {
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
}
def curPrice = null;
if(curPriceObj){
curPrice = curPriceObj.avg_price
}else{
curPrice = weekendPriceObj.avg_price
}
if(weekendPriceObj.value_type!=1){
if(dist.city == "福州"){
if(Math.abs(mom)*100>3){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}else{
if(Math.abs(mom)*100>10){
def randMarkup = nextDouble(0, 10)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100,4)
}else{
mom = DataUtil.roundDouble(randMarkup/100,4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}
}
double yoy = 0d
if(lastYearPriceObj){
yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4)
}
if(curPriceObj){
if(curPrice!=weekendPriceObj.avg_price){
def data = new BasicDBObject()
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
data.put("mom", mom)
data.put("yoy", yoy)
data.put("avg_price", curPrice)
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
}else{
def data = new BasicDBObject()
data.put("city", dist.city)
data.put("region", dist.region)
data.put("district_name", dist.name)
data.put("estate_type", dist.estate_type2)
data.put("date", curMonth)
data.put("avg_price", curPrice)
data.put("credit", weekendPriceObj.credit)
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
data.put("mom", mom)
data.put("yoy", yoy)
DISTRICT_SELL_STAT.save(data)
println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
//随机浮点数
public static double nextDouble(final double min, final double max) throws Exception {
DecimalFormat df = new DecimalFormat("#.00");
if (max < min) {
throw new Exception("min < max");
}
if (min == max) {
return min;
}
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
}
def findPriceMap(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DISTRICT_STAT.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.region +"##"+d.district_name, d)
}
return priceMap
}
def findMonthlyPrice(city, monDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("date", monDate)
def dbcursor = DISTRICT_SELL_STAT.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.region +"##"+d.district_name, d)
}
return priceMap
}
// def updateDistrictSellStat(citys){
// def index = 0
// citys.each{city->
// index ++
// def stats = FZ_DISTRICT_STAT.find(new BasicDBObject("month",202209).append("city", city).append("avg_price", new BasicDBObject(QueryOperators.GTE, 0))).asList()
// if(stats){
// batchUpdate(stats)
// }
// println "[${index}] "+city+" 完成"
// }
// }
//
// def batchUpdate(def stats){
// def totalCount = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(stats.size())
//
// for(def i=0;i<stats.size();i++){
// final def stat = stats[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
// totalCount.addAndGet(1)
// updateHistory(stat)
//// println totalCount.get() +" "+stat.city+" "+stat.name
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// };
// })
// }
// latch.await()
// }
//
// def updateHistory(stat){
// def obj = findDistrictSellStatByDate(stat.city, stat.region, stat.name, stat.month)
// if(obj){
// if(obj.avg_price!=stat.avg_price){
// def monDate = getMomDate(stat.month)
// def yoyDate = getYoyDate(stat.month)
//
// def monDbo = findDistrictSellStatByDate(stat.city, stat.region, stat.name, monDate)
// def yoyDbo = findDistrictSellStatByDate(stat.city, stat.region, stat.name, yoyDate)
// double mom = 0d
// if(monDbo && monDbo.avg_price != 0) {
// mom = DataUtil.roundDouble(((stat.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4)
// }
// double yoy = 0d
// if(yoyDbo != null && yoyDbo.avg_price != 0){
// yoy = DataUtil.roundDouble(((stat.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4)
// }
//
// def udata = new BasicDBObject();
// udata.put("avg_price", stat.avg_price)
// udata.put("mom", mom)
// udata.put("yoy", yoy)
// udata.put("m_date", new Date())
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", obj._id) , new BasicDBObject('$set', udata),true,false)
// }
// }else{
// println stat.city+" "+stat.region+" "+stat.name+" 不存在"
// }
// }
//
// def findDistrictSellStatByDate(city, region, name, month){
// def q = new BasicDBObject("city", city).append("region", region).append("district_name", name).append("date", month)
// def obj = DISTRICT_SELL_STAT.findOne(q)
// return obj;
// }
}
@@ -221,6 +221,15 @@ class CcbCodeUtils {
}
return "99"
}
static def getUseTypeCode(def estateType){
if(estateType=="商住"){
return "13"
}else if(estateType && (estateType=="普通住宅")){
return "11"
}
return "11"
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getHouseStructureCode(){
@@ -38,13 +38,13 @@ public class BeiKeCrawlByChrome {
def task = new BeiKeCrawlByChrome();
task.run()
// task.run1()
task.addDealRegion();
// task.addDealRegion();
}
def run(){
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
detail.total_price = price.toDouble() * 10000
}
if (detail.total_price && detail.area) {
@@ -398,13 +398,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("合肥")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
// }
}
}
}
return list
@@ -38,7 +38,7 @@ public class LianjiaCrawlByChrome {
task.run()
// task.run1()
// task.addDealRegion();
task.standardRegion()
// task.standardRegion()
// task.updadteDealPriceField()
}
@@ -117,7 +117,7 @@ public class LianjiaCrawlByChrome {
}
def run(){
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
detail.total_price = price.toDouble() * 10000
}
if (detail.total_price && detail.area) {
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
}
}
@@ -120,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0)
def final AtomicInteger successIndex = new AtomicInteger(0)
datas.each { dbo ->
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
fixedThreadPool.execute(
new Runnable() {
void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
if(dbo.name){
@@ -162,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
}finally {
latch.countDown()
}
// }
// }
// );
}
}
);
}
// latch.await()
latch.await()
long end = System.currentTimeMillis()
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
@@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京")
qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
@@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
def esSources = ["中原地产":"中原地产"]
def esSources = [ "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
static main(args) {
@@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
def citys = ["福州"]//CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202208);
def count2 = findTotalCount(source, 202209);
def count1 = findTotalCount(source, 202209);
def count2 = findTotalCount(source, 202210);
println source +" "+count1 +" "+count2
}
}