Merge branch 'script_gj' into 'master'
Script gj See merge request !431
This commit is contained in:
@@ -242,5 +242,13 @@
|
|||||||
<version>1.0.0</version>
|
<version>1.0.0</version>
|
||||||
<classifier>jdk18</classifier>
|
<classifier>jdk18</classifier>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>org.springframework.boot</groupId>-->
|
||||||
|
<!-- <artifactId>spring-boot-devtools</artifactId>-->
|
||||||
|
<!-- <!– optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 –>-->
|
||||||
|
<!-- <optional>true</optional>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
+22
-16
@@ -51,7 +51,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
|||||||
|
|
||||||
index ++
|
index ++
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
|
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||||
|
|
||||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
||||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||||
@@ -175,25 +175,31 @@ class BigdataCommunityBatch extends BaseBatch{
|
|||||||
ret << "pa101"
|
ret << "pa101"
|
||||||
ret << ""
|
ret << ""
|
||||||
if(dist.address){
|
if(dist.address){
|
||||||
def jsonObj = null
|
if(dist.street){
|
||||||
if(dist.address){
|
ret << dist.street
|
||||||
jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
if(dist.street_number){
|
||||||
}
|
ret << dist.street_number
|
||||||
if(jsonObj && jsonObj.result && jsonObj.result.street){
|
|
||||||
if(jsonObj.result.street){
|
|
||||||
ret << jsonObj.result.street
|
|
||||||
}else{
|
|
||||||
ret << ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if(jsonObj.result.streetNumber){
|
|
||||||
ret << jsonObj.result.streetNumber
|
|
||||||
}else{
|
}else{
|
||||||
ret << ""
|
ret << ""
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
ret << ""
|
def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||||
ret << ""
|
if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||||
|
if(jsonObj.result.street){
|
||||||
|
ret << jsonObj.result.street
|
||||||
|
}else{
|
||||||
|
ret << ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if(jsonObj.result.streetNumber){
|
||||||
|
ret << jsonObj.result.streetNumber
|
||||||
|
}else{
|
||||||
|
ret << ""
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
ret << ""
|
||||||
|
ret << ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
ret << ""
|
ret << ""
|
||||||
|
|||||||
+6
-5
@@ -33,8 +33,8 @@ import java.util.regex.Pattern
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class BigdataDealBatch extends BaseBatch{
|
class BigdataDealBatch extends BaseBatch{
|
||||||
|
|
||||||
// def startDate = 202206
|
def startDate = 202206
|
||||||
// def endDate = 202208
|
def endDate = 202208
|
||||||
|
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||||
@@ -58,11 +58,12 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||||
// def citys = CcbCommonUtils.citys;
|
// def citys = CcbCommonUtils.citys;
|
||||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
if(args && args.size()>=2){
|
if(args && args.size()>=2){
|
||||||
curDate = args[1].toString().toInteger()
|
curDate = args[1].toString().toInteger()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
println "curDate:"+curDate
|
||||||
// for(def city : citys){
|
// for(def city : citys){
|
||||||
//// if(!ccbCitys.contains(city)){
|
//// if(!ccbCitys.contains(city)){
|
||||||
// addDealPriceMatch(city);
|
// addDealPriceMatch(city);
|
||||||
@@ -110,9 +111,9 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
def totalCount = new AtomicLong()
|
def totalCount = new AtomicLong()
|
||||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||||
|
|
||||||
// def lianjiaPriceMap = findThirdPartyPrice("上海")
|
def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||||
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
||||||
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||||
for(def i=0;i<rets.size();i++){
|
for(def i=0;i<rets.size();i++){
|
||||||
final def trade = rets[i]
|
final def trade = rets[i]
|
||||||
fixedThreadPool.execute(new Runnable() {
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
|||||||
+1
-1
@@ -14,7 +14,7 @@ import java.sql.ResultSet
|
|||||||
|
|
||||||
class ExportOracleDataTest {
|
class ExportOracleDataTest {
|
||||||
|
|
||||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
|||||||
+39
@@ -39,6 +39,44 @@ class AddressCut extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
<<<<<<< HEAD
|
||||||
|
// sameAddr()
|
||||||
|
seaweedStreetUpdate()
|
||||||
|
}
|
||||||
|
|
||||||
|
def seaweedStreetUpdate(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
|
||||||
|
if(d['A']){
|
||||||
|
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||||
|
if(dbo){
|
||||||
|
def udate = new BasicDBObject()
|
||||||
|
def street = null
|
||||||
|
def address = ""
|
||||||
|
if(d['E']){
|
||||||
|
street = d['E']
|
||||||
|
udate.put("street", street)
|
||||||
|
address = street
|
||||||
|
}
|
||||||
|
def streetNum = null
|
||||||
|
if(d['F']){
|
||||||
|
streetNum = d['F']
|
||||||
|
udate.put("street_number", streetNum)
|
||||||
|
address = address + streetNum
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dbo.address!=address){
|
||||||
|
udate.put("address", address)
|
||||||
|
}
|
||||||
|
if(udate){
|
||||||
|
udate.put("m_date", new Date())
|
||||||
|
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
=======
|
||||||
// sameAddr() //相同地址小区
|
// sameAddr() //相同地址小区
|
||||||
// roadMatch()
|
// roadMatch()
|
||||||
addAnjukeRoad()
|
addAnjukeRoad()
|
||||||
@@ -93,6 +131,7 @@ class AddressCut extends BaseBatch{
|
|||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
|
||||||
}
|
}
|
||||||
|
|
||||||
def sameAddr() {
|
def sameAddr() {
|
||||||
|
|||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
package com.fangjia.script.ccbevaluation.ronghe
|
||||||
|
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
|
||||||
|
class BigdataCitypriceBatch extends BaseBatch{
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
runBatch(new BigdataCitypriceBatch(), args)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void process(Object args) {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
+52
-52
@@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
|||||||
// })
|
// })
|
||||||
// }
|
// }
|
||||||
|
|
||||||
addBigdataDistrictFJW()
|
// addBigdataDistrictFJW()
|
||||||
|
|
||||||
// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
||||||
|
|
||||||
|
def index = 0
|
||||||
|
for (def city : citys) {
|
||||||
|
index++
|
||||||
|
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||||
//
|
//
|
||||||
// def index = 0
|
// for(int i=0;i<10;i++){
|
||||||
// for (def city : citys) {
|
// if(districts.size()==0){
|
||||||
// index++
|
// Thread.sleep(1000*60)
|
||||||
// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||||
////
|
// }else{
|
||||||
//// for(int i=0;i<10;i++){
|
// break
|
||||||
//// if(districts.size()==0){
|
|
||||||
//// Thread.sleep(1000*60)
|
|
||||||
//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
|
||||||
//// }else{
|
|
||||||
//// break
|
|
||||||
//// }
|
|
||||||
//// }
|
|
||||||
//
|
|
||||||
// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
|
|
||||||
//
|
|
||||||
// for(def dist : districts){
|
|
||||||
// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
|
|
||||||
// if(obj){
|
|
||||||
// if(obj.alias && obj.alias.size()>0){
|
|
||||||
// dist.commalias = obj.alias.get(0)
|
|
||||||
// }else{
|
|
||||||
// dist.commalias = ""
|
|
||||||
// }
|
|
||||||
// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
|
||||||
// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
|
|
||||||
// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
|
|
||||||
// dist.houseuse = "11"
|
|
||||||
// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
|
|
||||||
// dist.compid = new ObjectId() //物业公司id
|
|
||||||
// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
|
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
// println index +" "+city +" 小区加载完成 "+districts.size()
|
|
||||||
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
|
||||||
// println index +" "+city +" 1.小区表完成"
|
|
||||||
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
for(def dist : districts){
|
||||||
// println index +" "+city +" 2.小区建筑结构表"
|
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
|
||||||
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
if(obj){
|
||||||
// println index +" "+city +" 3.小区房屋类型表"
|
if(obj.alias && obj.alias.size()>0){
|
||||||
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
dist.commalias = obj.alias.get(0)
|
||||||
// println index +" "+city +" 4.小区建筑年代表"
|
}else{
|
||||||
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
dist.commalias = ""
|
||||||
// println index +" "+city +" 5.小区费用表"
|
}
|
||||||
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||||
// println index +" "+city +" 6.小区房屋用途表"
|
dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
|
||||||
// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
|
||||||
// println index +" "+ city + " 7.地理位置表"
|
dist.houseuse = CcbCodeUtils.getUseTypeCode(obj.estate_type2)//"11"
|
||||||
//
|
dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||||
//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
dist.compid = new ObjectId() //物业公司id
|
||||||
//
|
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
|
||||||
// }
|
}
|
||||||
|
}
|
||||||
|
println index +" "+city +" 小区加载完成 "+districts.size()
|
||||||
|
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
||||||
|
println index +" "+city +" 1.小区表完成"
|
||||||
|
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
||||||
|
println index +" "+city +" 2.小区建筑结构表"
|
||||||
|
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
||||||
|
println index +" "+city +" 3.小区房屋类型表"
|
||||||
|
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
||||||
|
println index +" "+city +" 4.小区建筑年代表"
|
||||||
|
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
||||||
|
println index +" "+city +" 5.小区费用表"
|
||||||
|
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||||
|
println index +" "+city +" 6.小区房屋用途表"
|
||||||
|
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
||||||
|
println index +" "+ city + " 7.地理位置表"
|
||||||
|
|
||||||
|
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def addBigdataCommunityGJRH(def districts, def cityid){
|
def addBigdataCommunityGJRH(def districts, def cityid){
|
||||||
|
|||||||
+66
-41
@@ -31,6 +31,8 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new DistrictPriceExportBatch(), args)
|
runBatch(new DistrictPriceExportBatch(), args)
|
||||||
@@ -41,18 +43,8 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
|
|
||||||
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
||||||
|
|
||||||
def citys = CommonDatautils.findAllCity()
|
def ret = createNewDatas("福州")
|
||||||
citys.remove("福州");
|
saveToDB(ret)
|
||||||
citys.add(0, "福州")
|
|
||||||
|
|
||||||
def fz_datas = []
|
|
||||||
for(def city : citys){
|
|
||||||
def ret = createDatas("福州")
|
|
||||||
saveToDB(ret)
|
|
||||||
if(city == "福州"){
|
|
||||||
fz_datas = ret
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||||
@@ -60,7 +52,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||||
}
|
}
|
||||||
def sb = appendResult(headers, fz_datas)
|
def sb = appendResult(headers, ret)
|
||||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||||
FzFtpUtils.ftpUpload(fileName)
|
FzFtpUtils.ftpUpload(fileName)
|
||||||
//生成加密数据文件并上传
|
//生成加密数据文件并上传
|
||||||
@@ -106,16 +98,50 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
return rets
|
return rets
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def createNewDatas(def city){
|
||||||
|
def districts = getFzDistricts(city)
|
||||||
|
def curPriceMap = findFzDistrictSellStat("福州",curDate)
|
||||||
|
def result = []
|
||||||
|
for(def rMap : districts){
|
||||||
|
def curPrice = 0
|
||||||
|
def dbo = null
|
||||||
|
if(curPriceMap.containsKey(rMap.get("_id").toString())){
|
||||||
|
dbo = curPriceMap.get(rMap.get("_id").toString())
|
||||||
|
curPrice = dbo.avg_price
|
||||||
|
}else{
|
||||||
|
curPrice = specialDistsPriceMap.get(rMap.name)
|
||||||
|
if(curPrice){
|
||||||
|
def randMarkup = nextDouble(0, 1)
|
||||||
|
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def m = []
|
||||||
|
if(curPrice && dbo){
|
||||||
|
m << dbo.get("_id").toString()//均价ID
|
||||||
|
}else{
|
||||||
|
m << new ObjectId().toString()
|
||||||
|
}
|
||||||
|
m << rMap.get("_id").toString()//楼盘ID
|
||||||
|
m << rMap.city
|
||||||
|
m << rMap.region
|
||||||
|
m << rMap.name
|
||||||
|
if(curPrice){
|
||||||
|
m << curPrice
|
||||||
|
}else{
|
||||||
|
m << 0
|
||||||
|
}
|
||||||
|
m << curDate.toInteger();
|
||||||
|
result << m
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
def createDatas(def city){
|
def createDatas(def city){
|
||||||
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||||
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||||
// districts.addAll(removeDists)
|
// districts.addAll(removeDists)
|
||||||
def districts = []
|
def districts = getFzDistricts(city)
|
||||||
if(city == "福州"){
|
|
||||||
districts = getFzDistricts(city)
|
|
||||||
}else{
|
|
||||||
districts = CommonDatautils.findAllDistricts(city, "district", null);
|
|
||||||
}
|
|
||||||
|
|
||||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||||
def dataStr = DateUtils.add(endDateStr, -30)
|
def dataStr = DateUtils.add(endDateStr, -30)
|
||||||
@@ -128,38 +154,27 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||||
// map2.put(key, row[5])
|
// map2.put(key, row[5])
|
||||||
// }
|
// }
|
||||||
|
|
||||||
def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate)
|
def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate)
|
||||||
|
|
||||||
def result = []
|
def result = []
|
||||||
|
|
||||||
for(def rMap : districts){
|
for(def rMap : districts){
|
||||||
|
|
||||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, curDate)
|
||||||
def curPrice = 0
|
def curPrice = 0
|
||||||
if(dbo){
|
if(dbo){
|
||||||
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
||||||
curPrice = dbo.get("avg_price")
|
curPrice = dbo.get("avg_price")
|
||||||
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
||||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||||
if(city == "福州"){
|
if(Math.abs(markup)>3){
|
||||||
if(Math.abs(markup)>3){
|
def randMarkup = nextDouble(0, 3)
|
||||||
def randMarkup = nextDouble(0, 3)
|
if(markup < 0){
|
||||||
if(markup < 0){
|
markup = randMarkup*(-1)
|
||||||
markup = randMarkup*(-1)
|
}else{
|
||||||
}else{
|
markup = randMarkup
|
||||||
markup = randMarkup
|
|
||||||
}
|
|
||||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
|
||||||
}
|
|
||||||
}else{
|
|
||||||
if(Math.abs(markup)>10){
|
|
||||||
def randMarkup = nextDouble(0, 10)
|
|
||||||
if(markup < 0){
|
|
||||||
markup = randMarkup*(-10)
|
|
||||||
}else{
|
|
||||||
markup = randMarkup
|
|
||||||
}
|
|
||||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
|
||||||
}
|
}
|
||||||
|
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
@@ -190,6 +205,17 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def findFzDistrictSellStat(city,lastMonthDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("date", lastMonthDate)
|
||||||
|
def priceMap = [:]
|
||||||
|
DISTRICT_SELL_STAT.find(q).each{
|
||||||
|
priceMap.put(it.commid, it)
|
||||||
|
}
|
||||||
|
return priceMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def findFzDistrictStat(city,lastMonthDate){
|
def findFzDistrictStat(city,lastMonthDate){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
@@ -256,7 +282,6 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
udata.put("avg_price", r[5])
|
udata.put("avg_price", r[5])
|
||||||
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
||||||
}else{
|
}else{
|
||||||
def udata = new BasicDBObject()
|
|
||||||
def o = new BasicDBObject()
|
def o = new BasicDBObject()
|
||||||
println r[0]
|
println r[0]
|
||||||
o.put("_id", new ObjectId(r[0]))
|
o.put("_id", new ObjectId(r[0]))
|
||||||
|
|||||||
+321
@@ -0,0 +1,321 @@
|
|||||||
|
package com.fangjia.script.fuzhou
|
||||||
|
|
||||||
|
import com.fangjia.core.util.DataUtil
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.BatchOwner
|
||||||
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.QueryOperators
|
||||||
|
|
||||||
|
import java.text.DecimalFormat
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
|
class DistrictSellStatBatch extends BaseBatch {
|
||||||
|
|
||||||
|
static DBCollection FZ_DISTRICT_STAT = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
static DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||||
|
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||||
|
static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
runBatch(new DistrictSellStatBatch(), args);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void process(Object args) {
|
||||||
|
calcMonthlyPrice()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getMomDate(int date){
|
||||||
|
|
||||||
|
Calendar calendar = Calendar.getInstance()
|
||||||
|
calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
|
||||||
|
calendar.add(Calendar.MONTH, -1)
|
||||||
|
return calendar.getTime().format("yyyyMM").toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
def getYoyDate(int date){
|
||||||
|
|
||||||
|
Calendar calendar = Calendar.getInstance()
|
||||||
|
calendar.setTime(Date.parse("yyyyMM", String.valueOf(date)))
|
||||||
|
calendar.add(Calendar.YEAR, -1)
|
||||||
|
return calendar.getTime().format("yyyyMM").toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
def calcMonthlyPrice(){
|
||||||
|
def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity()
|
||||||
|
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||||
|
|
||||||
|
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
||||||
|
|
||||||
|
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
|
def monDate = getMomDate(curMonth)
|
||||||
|
def yoyDate = getYoyDate(curMonth)
|
||||||
|
|
||||||
|
def index = 0
|
||||||
|
for(def city : citys){
|
||||||
|
|
||||||
|
index++
|
||||||
|
def districts = []
|
||||||
|
if(city == "福州"){
|
||||||
|
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||||
|
}else{
|
||||||
|
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
|
||||||
|
}
|
||||||
|
|
||||||
|
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
|
||||||
|
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||||
|
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
|
||||||
|
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
|
||||||
|
if(districts && weekendPriceMap){
|
||||||
|
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
|
||||||
|
}
|
||||||
|
println index+" "+city +" "+ (new Date()).toString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def running(districts){
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
|
districts.each{ dist->
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
println totalCount.get()+" "+ dist.city+" "+dist.name
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap){
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
|
districts.each{dist->
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def key = dist.region +"##" + dist.name
|
||||||
|
def weekendPriceObj = weekendPriceMap.get(key)
|
||||||
|
if(weekendPriceObj){
|
||||||
|
def curPriceObj = curMonthlyPriceMap.get(key)
|
||||||
|
def lastMonthPriceObj = lastMonthlyPriceMap.get(key)
|
||||||
|
def lastYearPriceObj = lastYearPriceMap.get(key)
|
||||||
|
double mom = 0d
|
||||||
|
|
||||||
|
if(lastMonthPriceObj) {
|
||||||
|
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
|
||||||
|
}
|
||||||
|
|
||||||
|
def curPrice = null;
|
||||||
|
if(curPriceObj){
|
||||||
|
curPrice = curPriceObj.avg_price
|
||||||
|
}else{
|
||||||
|
curPrice = weekendPriceObj.avg_price
|
||||||
|
}
|
||||||
|
|
||||||
|
if(weekendPriceObj.value_type!=1){
|
||||||
|
if(dist.city == "福州"){
|
||||||
|
if(Math.abs(mom)*100>3){
|
||||||
|
def randMarkup = nextDouble(0, 3)
|
||||||
|
if(mom < 0){
|
||||||
|
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||||
|
}else{
|
||||||
|
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||||
|
}
|
||||||
|
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(Math.abs(mom)*100>10){
|
||||||
|
def randMarkup = nextDouble(0, 10)
|
||||||
|
if(mom < 0){
|
||||||
|
mom = DataUtil.roundDouble(randMarkup*(-1)/100,4)
|
||||||
|
}else{
|
||||||
|
mom = DataUtil.roundDouble(randMarkup/100,4)
|
||||||
|
}
|
||||||
|
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
double yoy = 0d
|
||||||
|
if(lastYearPriceObj){
|
||||||
|
yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4)
|
||||||
|
}
|
||||||
|
if(curPriceObj){
|
||||||
|
if(curPrice!=weekendPriceObj.avg_price){
|
||||||
|
def data = new BasicDBObject()
|
||||||
|
data.put("m_date", new Date())
|
||||||
|
data.put("commid", dist._id.toString())
|
||||||
|
data.put("mom", mom)
|
||||||
|
data.put("yoy", yoy)
|
||||||
|
data.put("avg_price", curPrice)
|
||||||
|
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||||
|
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
def data = new BasicDBObject()
|
||||||
|
data.put("city", dist.city)
|
||||||
|
data.put("region", dist.region)
|
||||||
|
data.put("district_name", dist.name)
|
||||||
|
data.put("estate_type", dist.estate_type2)
|
||||||
|
data.put("date", curMonth)
|
||||||
|
data.put("avg_price", curPrice)
|
||||||
|
data.put("credit", weekendPriceObj.credit)
|
||||||
|
data.put("m_date", new Date())
|
||||||
|
data.put("commid", dist._id.toString())
|
||||||
|
data.put("mom", mom)
|
||||||
|
data.put("yoy", yoy)
|
||||||
|
DISTRICT_SELL_STAT.save(data)
|
||||||
|
println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//随机浮点数
|
||||||
|
public static double nextDouble(final double min, final double max) throws Exception {
|
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat("#.00");
|
||||||
|
if (max < min) {
|
||||||
|
throw new Exception("min < max");
|
||||||
|
}
|
||||||
|
if (min == max) {
|
||||||
|
return min;
|
||||||
|
}
|
||||||
|
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findPriceMap(city, weekend){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("weekend", weekend)
|
||||||
|
def dbcursor = DISTRICT_STAT.find(q)
|
||||||
|
def priceMap = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
priceMap.put(d.region +"##"+d.district_name, d)
|
||||||
|
}
|
||||||
|
return priceMap
|
||||||
|
}
|
||||||
|
|
||||||
|
def findMonthlyPrice(city, monDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("date", monDate)
|
||||||
|
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
||||||
|
def priceMap = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
priceMap.put(d.region +"##"+d.district_name, d)
|
||||||
|
}
|
||||||
|
return priceMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// def updateDistrictSellStat(citys){
|
||||||
|
// def index = 0
|
||||||
|
// citys.each{city->
|
||||||
|
// index ++
|
||||||
|
// def stats = FZ_DISTRICT_STAT.find(new BasicDBObject("month",202209).append("city", city).append("avg_price", new BasicDBObject(QueryOperators.GTE, 0))).asList()
|
||||||
|
// if(stats){
|
||||||
|
// batchUpdate(stats)
|
||||||
|
// }
|
||||||
|
// println "[${index}] "+city+" 完成"
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def batchUpdate(def stats){
|
||||||
|
// def totalCount = new AtomicLong()
|
||||||
|
// final CountDownLatch latch = new CountDownLatch(stats.size())
|
||||||
|
//
|
||||||
|
// for(def i=0;i<stats.size();i++){
|
||||||
|
// final def stat = stats[i]
|
||||||
|
// fixedThreadPool.execute(new Runnable() {
|
||||||
|
// void run() {
|
||||||
|
// try {
|
||||||
|
// totalCount.addAndGet(1)
|
||||||
|
// updateHistory(stat)
|
||||||
|
//// println totalCount.get() +" "+stat.city+" "+stat.name
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// e.printStackTrace()
|
||||||
|
// } finally {
|
||||||
|
// latch.countDown()
|
||||||
|
// }
|
||||||
|
// };
|
||||||
|
// })
|
||||||
|
// }
|
||||||
|
// latch.await()
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def updateHistory(stat){
|
||||||
|
// def obj = findDistrictSellStatByDate(stat.city, stat.region, stat.name, stat.month)
|
||||||
|
// if(obj){
|
||||||
|
// if(obj.avg_price!=stat.avg_price){
|
||||||
|
// def monDate = getMomDate(stat.month)
|
||||||
|
// def yoyDate = getYoyDate(stat.month)
|
||||||
|
//
|
||||||
|
// def monDbo = findDistrictSellStatByDate(stat.city, stat.region, stat.name, monDate)
|
||||||
|
// def yoyDbo = findDistrictSellStatByDate(stat.city, stat.region, stat.name, yoyDate)
|
||||||
|
// double mom = 0d
|
||||||
|
// if(monDbo && monDbo.avg_price != 0) {
|
||||||
|
// mom = DataUtil.roundDouble(((stat.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4)
|
||||||
|
// }
|
||||||
|
// double yoy = 0d
|
||||||
|
// if(yoyDbo != null && yoyDbo.avg_price != 0){
|
||||||
|
// yoy = DataUtil.roundDouble(((stat.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def udata = new BasicDBObject();
|
||||||
|
// udata.put("avg_price", stat.avg_price)
|
||||||
|
// udata.put("mom", mom)
|
||||||
|
// udata.put("yoy", yoy)
|
||||||
|
// udata.put("m_date", new Date())
|
||||||
|
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", obj._id) , new BasicDBObject('$set', udata),true,false)
|
||||||
|
// }
|
||||||
|
// }else{
|
||||||
|
// println stat.city+" "+stat.region+" "+stat.name+" 不存在"
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def findDistrictSellStatByDate(city, region, name, month){
|
||||||
|
// def q = new BasicDBObject("city", city).append("region", region).append("district_name", name).append("date", month)
|
||||||
|
// def obj = DISTRICT_SELL_STAT.findOne(q)
|
||||||
|
// return obj;
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -221,6 +221,15 @@ class CcbCodeUtils {
|
|||||||
}
|
}
|
||||||
return "99"
|
return "99"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def getUseTypeCode(def estateType){
|
||||||
|
if(estateType=="商住"){
|
||||||
|
return "13"
|
||||||
|
}else if(estateType && (estateType=="普通住宅")){
|
||||||
|
return "11"
|
||||||
|
}
|
||||||
|
return "11"
|
||||||
|
}
|
||||||
//室内结构
|
//室内结构
|
||||||
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
||||||
static def getHouseStructureCode(){
|
static def getHouseStructureCode(){
|
||||||
|
|||||||
+5
-5
@@ -38,13 +38,13 @@ public class BeiKeCrawlByChrome {
|
|||||||
def task = new BeiKeCrawlByChrome();
|
def task = new BeiKeCrawlByChrome();
|
||||||
task.run()
|
task.run()
|
||||||
// task.run1()
|
// task.run1()
|
||||||
task.addDealRegion();
|
// task.addDealRegion();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
System.setProperty("http.nonProxyHosts", "localhost");
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
|
|
||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
|
|
||||||
def price = e.select(".totalPrice .number").text()
|
def price = e.select(".totalPrice .number").text()
|
||||||
if (price && price.isNumber()) {
|
if (price && price.isNumber()) {
|
||||||
detail.total_price = price.toInteger() * 10000
|
detail.total_price = price.toDouble() * 10000
|
||||||
}
|
}
|
||||||
|
|
||||||
if (detail.total_price && detail.area) {
|
if (detail.total_price && detail.area) {
|
||||||
@@ -398,13 +398,13 @@ public class BeiKeCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
// if(e.text().contains("合肥")){
|
if(e.text().contains("福州")){
|
||||||
def cityMap = [:]
|
def cityMap = [:]
|
||||||
cityMap.city = e.text()
|
cityMap.city = e.text()
|
||||||
cityMap.url = "https:"+href
|
cityMap.url = "https:"+href
|
||||||
list << cityMap
|
list << cityMap
|
||||||
// cityMap.put(e.text(), "https:"+href)
|
// cityMap.put(e.text(), "https:"+href)
|
||||||
// }
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return list
|
return list
|
||||||
|
|||||||
+4
-4
@@ -38,7 +38,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
task.run()
|
task.run()
|
||||||
// task.run1()
|
// task.run1()
|
||||||
// task.addDealRegion();
|
// task.addDealRegion();
|
||||||
task.standardRegion()
|
// task.standardRegion()
|
||||||
// task.updadteDealPriceField()
|
// task.updadteDealPriceField()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,7 +117,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
System.setProperty("http.nonProxyHosts", "localhost");
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
def cityMap = getAllCityUrls(driver)
|
def cityMap = getAllCityUrls(driver)
|
||||||
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
|
|
||||||
def price = e.select(".totalPrice .number").text()
|
def price = e.select(".totalPrice .number").text()
|
||||||
if (price && price.isNumber()) {
|
if (price && price.isNumber()) {
|
||||||
detail.total_price = price.toInteger() * 10000
|
detail.total_price = price.toDouble() * 10000
|
||||||
}
|
}
|
||||||
|
|
||||||
if (detail.total_price && detail.area) {
|
if (detail.total_price && detail.area) {
|
||||||
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if (!href.contains("fang.lianjia")){
|
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||||
cityMap.put(e.text(), href)
|
cityMap.put(e.text(), href)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-8
@@ -120,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||||
def final AtomicInteger successIndex = new AtomicInteger(0)
|
def final AtomicInteger successIndex = new AtomicInteger(0)
|
||||||
datas.each { dbo ->
|
datas.each { dbo ->
|
||||||
// fixedThreadPool.execute(
|
fixedThreadPool.execute(
|
||||||
// new Runnable() {
|
new Runnable() {
|
||||||
// void run() {
|
void run() {
|
||||||
try{
|
try{
|
||||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||||
if(dbo.name){
|
if(dbo.name){
|
||||||
@@ -162,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}finally {
|
}finally {
|
||||||
latch.countDown()
|
latch.countDown()
|
||||||
}
|
}
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
// );
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// latch.await()
|
latch.await()
|
||||||
long end = System.currentTimeMillis()
|
long end = System.currentTimeMillis()
|
||||||
|
|
||||||
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||||
@@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def findAllNoMatchData(){
|
def findAllNoMatchData(){
|
||||||
DBObject qdata = new BasicDBObject();
|
DBObject qdata = new BasicDBObject();
|
||||||
qdata.put("s_date", SHORT_DATE)
|
qdata.put("s_date", SHORT_DATE)
|
||||||
// qdata.put("city","北京")
|
qdata.put("city","福州")
|
||||||
// qdata.put("source","安居客")
|
// qdata.put("source","安居客")
|
||||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||||
|
|||||||
+2
-2
@@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||||
|
|
||||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
|
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
|
||||||
def esSources = ["中原地产":"中原地产"]
|
def esSources = [ "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
|
||||||
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
@@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
|
|
||||||
println "starting......"
|
println "starting......"
|
||||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||||
def citys = CommonUtils.findAllCity()
|
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||||
|
|
||||||
esSources.each{esSource, mgSource->
|
esSources.each{esSource, mgSource->
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
|
|||||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
|||||||
def run(){
|
def run(){
|
||||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||||
for(def source :sources){
|
for(def source :sources){
|
||||||
def count1 = findTotalCount(source, 202208);
|
def count1 = findTotalCount(source, 202209);
|
||||||
def count2 = findTotalCount(source, 202209);
|
def count2 = findTotalCount(source, 202210);
|
||||||
println source +" "+count1 +" "+count2
|
println source +" "+count1 +" "+count2
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user