按小区存房源
This commit is contained in:
@@ -530,7 +530,13 @@
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>com.fangjia</groupId>-->
|
||||
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||
<!-- <version>0.0.1-SNAPSHOT</version>-->
|
||||
<!-- <scope>compile</scope>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
|
||||
</dependencies>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+340
-3
@@ -1,13 +1,350 @@
|
||||
package com.fangjia.script.sync
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.MaJinTao)
|
||||
class CommunityHouseSampleBatch extends BaseBatch{
|
||||
class CommunityHouseSampleBatch extends BaseBatch {
|
||||
|
||||
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
|
||||
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new CommunityHouseSampleBatch(), args)
|
||||
}
|
||||
|
||||
def findHouseCase(Object district_stat, Map<String, List> houseMap) {
|
||||
// 查本小区的房源
|
||||
def houseQuery = new BasicDBObject()
|
||||
houseQuery.put("cat", "sell")
|
||||
houseQuery.put("city", district_stat.city)
|
||||
houseQuery.put("region", district_stat.region)
|
||||
houseQuery.put("district_name", district_stat.district_name)
|
||||
def houseListAll = SEABIRD.find(houseQuery).toList()
|
||||
|
||||
// 将均价符合的房源放入结果集
|
||||
def houseList = houseListAll.findAll() { housecase ->
|
||||
def totalPrice = housecase.total_price
|
||||
def houseArea = housecase.area
|
||||
def houseAvgPrice = totalPrice / houseArea
|
||||
def avgPrice = district_stat.avg_price
|
||||
houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2
|
||||
}
|
||||
|
||||
// 将样本数据按面积段分,200以下每10为一段,200以上每30为一段
|
||||
// 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20
|
||||
for (def housecase : houseList) {
|
||||
if (housecase.area <= 200) {
|
||||
def area = housecase.area.toInteger()
|
||||
def key = "g" + (area / 10).toInteger() * 10
|
||||
def gList = houseMap.get(key)
|
||||
|
||||
if (gList == null || gList.size() == 0) {
|
||||
def gHouseCaseList = []
|
||||
gHouseCaseList.add(housecase)
|
||||
houseMap.put(key, gHouseCaseList)
|
||||
} else if (gList.size() < 10) {
|
||||
gList.add(housecase)
|
||||
houseMap.put(key, gList)
|
||||
}
|
||||
} else {
|
||||
def area = housecase.area.toInteger()
|
||||
def startPoint = 200
|
||||
def segmentSize = 30
|
||||
def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger()
|
||||
def key = "g" + rangeStart
|
||||
def gList = houseMap.get(key)
|
||||
|
||||
if (gList == null || gList.size() == 0) {
|
||||
def gHouseCaseList = []
|
||||
gHouseCaseList.add(housecase)
|
||||
houseMap.put(key, gHouseCaseList)
|
||||
} else if (gList.size() < 10) {
|
||||
gList.add(housecase)
|
||||
houseMap.put(key, gList)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(5)
|
||||
|
||||
try {
|
||||
def cityList = CommonUtils.findAllCity()
|
||||
|
||||
// cityList = cityList.subList(0,150)
|
||||
cityList.each { city ->
|
||||
executorService.submit({
|
||||
def count = 0
|
||||
def documents = []
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||
def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend)
|
||||
List<BasicDBObject> districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||
|
||||
// 测试,只选一个
|
||||
// districts = districts.subList(0, 1)
|
||||
|
||||
districts.each { district ->
|
||||
Map<String, List> houseMap = [:]
|
||||
def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name)
|
||||
|
||||
if (district_stat != null) {
|
||||
findHouseCase(district_stat, houseMap)
|
||||
}
|
||||
if (houseMap.size() == 0) {
|
||||
def aroundDistrictList = DbUtils.findAroundDistrictByRadius(district, 3000)
|
||||
// def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll
|
||||
aroundDistrictList.each { aroundDistrict ->
|
||||
def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name)
|
||||
if (aroundDistrict_stat != null) {
|
||||
findHouseCase(aroundDistrict_stat, houseMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def document = new BasicDBObject()
|
||||
document.put("city", district.city)
|
||||
document.put("region", district.region)
|
||||
document.put("district_name", district.name)
|
||||
document.put("house_case", houseMap)
|
||||
|
||||
documents.add(document)
|
||||
count++
|
||||
if (count % 10 == 0) {
|
||||
println(city + "已处理小区数----------------------------------------------------" + count)
|
||||
}
|
||||
}
|
||||
HOUSE_CASE.insert(documents)
|
||||
} as Runnable)
|
||||
}
|
||||
} finally {
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
package com.fangjia.script.sync
|
||||
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
|
||||
@BatchOwner(Watcher.MaJinTao)
|
||||
class CommunityHouseSampleBatch extends BaseBatch {
|
||||
|
||||
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||
|
||||
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
|
||||
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
|
||||
|
||||
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017)
|
||||
|
||||
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new CommunityHouseSampleBatch(), args)
|
||||
}
|
||||
|
||||
def findHouseCase(Object district_stat, Map<String, List> houseMap) {
|
||||
|
||||
// 查本小区的房源
|
||||
def houseQuery = new BasicDBObject()
|
||||
//houseQuery.put("city", "深圳")
|
||||
|
||||
//houseQuery.put("region", "福田")
|
||||
|
||||
//houseQuery.put("district_name", "中泰燕南名庭")
|
||||
|
||||
houseQuery.put("cat", "sell")
|
||||
|
||||
houseQuery.put("city", district_stat.city)
|
||||
|
||||
houseQuery.put("region", district_stat.region)
|
||||
|
||||
houseQuery.put("district_name", district_stat.district_name)
|
||||
|
||||
def houseListAll = SEABIRD.find(houseQuery).toList()
|
||||
|
||||
// 将均价符合的房源放入结果集
|
||||
def houseList = houseListAll.findAll() { housecase ->
|
||||
def totalPrice = housecase.total_price
|
||||
def houseArea = housecase.area
|
||||
def houseAvgPrice = totalPrice / houseArea
|
||||
def avgPrice = district_stat.avg_price
|
||||
if (houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2) {
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 将样本数据按面积段分,200以下每10为一段,200以上每30为一段
|
||||
// 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20
|
||||
|
||||
for (def housecase : houseList) {
|
||||
if (housecase.area <= 200) {
|
||||
def area = housecase.area.toInteger()
|
||||
|
||||
def key = "g" + (area / 10).toInteger() * 10
|
||||
// println(key)
|
||||
// 获取该面积段的房源列表
|
||||
def gList = houseMap.get(key)
|
||||
|
||||
// 如果为空,说明该面积段没有房源,新建一个list,放入
|
||||
if (gList == null || gList.size() == 0) {
|
||||
def gHouseCaseList = []
|
||||
gHouseCaseList.add(housecase)
|
||||
houseMap.put(key, gHouseCaseList)
|
||||
} else {
|
||||
// 如果该面积段已经有10个房源,就不再放入
|
||||
if (gList.size() < 10) {
|
||||
// 如果不为空,说明该面积段已经有房源,直接放入
|
||||
gList.add(housecase)
|
||||
houseMap.put(key, gList)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
def area = housecase.area.toInteger()
|
||||
def startPoint = 200
|
||||
def segmentSize = 30
|
||||
def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger()
|
||||
def key = "g" + rangeStart
|
||||
// println(key)
|
||||
// 获取该面积段的房源列表
|
||||
def gList = houseMap.get(key)
|
||||
|
||||
// 如果为空,说明该面积段没有房源,新建一个list,放入
|
||||
if (gList == null || gList.size() == 0) {
|
||||
def gHouseCaseList = []
|
||||
gHouseCaseList.add(housecase)
|
||||
houseMap.put(key, gHouseCaseList)
|
||||
} else {
|
||||
// 如果该面积段已经有10个房源,就不再放入
|
||||
if (gList.size() < 10) {
|
||||
gList.add(housecase)
|
||||
houseMap.put(key, gList)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
|
||||
def count = 0
|
||||
|
||||
|
||||
def cityList = CommonUtils.findAllCity()
|
||||
|
||||
// 测试用,先只取一条
|
||||
// cityList = cityList.subList(0, 1)
|
||||
|
||||
print(cityList.size())
|
||||
print(cityList.toString())
|
||||
// cityList.each { city ->
|
||||
def city = cityList.get(0)
|
||||
print(city)
|
||||
def documents = []
|
||||
//查询该城市的所有小区
|
||||
List<BasicDBObject> districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||
|
||||
//测试用,先只取5个小区
|
||||
// districts = districts.subList(0, 2)
|
||||
|
||||
// 查询所有的小区均价
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend)
|
||||
for (def district : districts) {
|
||||
Map<String, List> houseMap = [:]
|
||||
|
||||
|
||||
|
||||
def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name)
|
||||
// 拿到小区对应的均价之后,去查房源
|
||||
if (district_stat != null) {
|
||||
findHouseCase(district_stat, houseMap)
|
||||
|
||||
}
|
||||
// 没有拿到小区对应的均价,找小区周边的三十个小区
|
||||
|
||||
def aroundDistrictListAll = DbUtils.findAroundDistrictByRadius(district, 3000)
|
||||
def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll
|
||||
|
||||
|
||||
// 遍历周边小区,找到均价符合的房源
|
||||
|
||||
aroundDistrictList.each { aroundDistrict ->
|
||||
def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name)
|
||||
// 拿到小区对应的均价之后,去查房源
|
||||
if (aroundDistrict_stat != null) {
|
||||
findHouseCase(aroundDistrict_stat, houseMap)
|
||||
}
|
||||
}
|
||||
print(houseMap)
|
||||
|
||||
// 创建一个新的文档
|
||||
def document = new BasicDBObject()
|
||||
|
||||
// 将 houseMap 和 communityStr 放入文档
|
||||
document.put("city", district.city)
|
||||
document.put("region", district.region)
|
||||
document.put("district_name", district.name)
|
||||
document.put("house_case", houseMap)
|
||||
|
||||
// 将文档插入到 MongoDB 中
|
||||
// HOUSE_CASE.in(document)
|
||||
|
||||
documents.add(document)
|
||||
count++
|
||||
if (count % 10 == 0) {
|
||||
print("已处理小区数----------------------------------------------------" + count)
|
||||
}
|
||||
}
|
||||
HOUSE_CASE.insert(documents)
|
||||
|
||||
}
|
||||
|
||||
// }
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user