按小区存房源
This commit is contained in:
@@ -530,7 +530,13 @@
|
|||||||
</exclusion>
|
</exclusion>
|
||||||
</exclusions>
|
</exclusions>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>com.fangjia</groupId>-->
|
||||||
|
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||||
|
<!-- <version>0.0.1-SNAPSHOT</version>-->
|
||||||
|
<!-- <scope>compile</scope>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
+340
-3
@@ -1,13 +1,350 @@
|
|||||||
package com.fangjia.script.sync
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import com.fangjia.datastore.driver.DSCollection
|
||||||
|
import com.fangjia.datastore.driver.DataStore
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
@BatchOwner(Watcher.MaJinTao)
|
@BatchOwner(Watcher.MaJinTao)
|
||||||
class CommunityHouseSampleBatch extends BaseBatch{
|
class CommunityHouseSampleBatch extends BaseBatch {
|
||||||
|
|
||||||
|
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||||
|
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||||
|
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
|
||||||
|
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||||
|
|
||||||
|
static void main(String[] args) {
|
||||||
|
runBatch(new CommunityHouseSampleBatch(), args)
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHouseCase(Object district_stat, Map<String, List> houseMap) {
|
||||||
|
// 查本小区的房源
|
||||||
|
def houseQuery = new BasicDBObject()
|
||||||
|
houseQuery.put("cat", "sell")
|
||||||
|
houseQuery.put("city", district_stat.city)
|
||||||
|
houseQuery.put("region", district_stat.region)
|
||||||
|
houseQuery.put("district_name", district_stat.district_name)
|
||||||
|
def houseListAll = SEABIRD.find(houseQuery).toList()
|
||||||
|
|
||||||
|
// 将均价符合的房源放入结果集
|
||||||
|
def houseList = houseListAll.findAll() { housecase ->
|
||||||
|
def totalPrice = housecase.total_price
|
||||||
|
def houseArea = housecase.area
|
||||||
|
def houseAvgPrice = totalPrice / houseArea
|
||||||
|
def avgPrice = district_stat.avg_price
|
||||||
|
houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2
|
||||||
|
}
|
||||||
|
|
||||||
|
// 将样本数据按面积段分,200以下每10为一段,200以上每30为一段
|
||||||
|
// 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20
|
||||||
|
for (def housecase : houseList) {
|
||||||
|
if (housecase.area <= 200) {
|
||||||
|
def area = housecase.area.toInteger()
|
||||||
|
def key = "g" + (area / 10).toInteger() * 10
|
||||||
|
def gList = houseMap.get(key)
|
||||||
|
|
||||||
|
if (gList == null || gList.size() == 0) {
|
||||||
|
def gHouseCaseList = []
|
||||||
|
gHouseCaseList.add(housecase)
|
||||||
|
houseMap.put(key, gHouseCaseList)
|
||||||
|
} else if (gList.size() < 10) {
|
||||||
|
gList.add(housecase)
|
||||||
|
houseMap.put(key, gList)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
def area = housecase.area.toInteger()
|
||||||
|
def startPoint = 200
|
||||||
|
def segmentSize = 30
|
||||||
|
def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger()
|
||||||
|
def key = "g" + rangeStart
|
||||||
|
def gList = houseMap.get(key)
|
||||||
|
|
||||||
|
if (gList == null || gList.size() == 0) {
|
||||||
|
def gHouseCaseList = []
|
||||||
|
gHouseCaseList.add(housecase)
|
||||||
|
houseMap.put(key, gHouseCaseList)
|
||||||
|
} else if (gList.size() < 10) {
|
||||||
|
gList.add(housecase)
|
||||||
|
houseMap.put(key, gList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void process(Object args) {
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(5)
|
||||||
|
|
||||||
|
try {
|
||||||
|
def cityList = CommonUtils.findAllCity()
|
||||||
|
|
||||||
|
// cityList = cityList.subList(0,150)
|
||||||
|
cityList.each { city ->
|
||||||
|
executorService.submit({
|
||||||
|
def count = 0
|
||||||
|
def documents = []
|
||||||
|
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||||
|
def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend)
|
||||||
|
List<BasicDBObject> districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||||
|
|
||||||
|
// 测试,只选一个
|
||||||
|
// districts = districts.subList(0, 1)
|
||||||
|
|
||||||
|
districts.each { district ->
|
||||||
|
Map<String, List> houseMap = [:]
|
||||||
|
def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name)
|
||||||
|
|
||||||
|
if (district_stat != null) {
|
||||||
|
findHouseCase(district_stat, houseMap)
|
||||||
|
}
|
||||||
|
if (houseMap.size() == 0) {
|
||||||
|
def aroundDistrictList = DbUtils.findAroundDistrictByRadius(district, 3000)
|
||||||
|
// def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll
|
||||||
|
aroundDistrictList.each { aroundDistrict ->
|
||||||
|
def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name)
|
||||||
|
if (aroundDistrict_stat != null) {
|
||||||
|
findHouseCase(aroundDistrict_stat, houseMap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def document = new BasicDBObject()
|
||||||
|
document.put("city", district.city)
|
||||||
|
document.put("region", district.region)
|
||||||
|
document.put("district_name", district.name)
|
||||||
|
document.put("house_case", houseMap)
|
||||||
|
|
||||||
|
documents.add(document)
|
||||||
|
count++
|
||||||
|
if (count % 10 == 0) {
|
||||||
|
println(city + "已处理小区数----------------------------------------------------" + count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HOUSE_CASE.insert(documents)
|
||||||
|
} as Runnable)
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
executorService.shutdown()
|
||||||
|
executorService.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
package com.fangjia.script.sync
|
||||||
|
|
||||||
|
import com.fangjia.datastore.driver.DSCollection
|
||||||
|
import com.fangjia.datastore.driver.DataStore
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.BatchOwner
|
||||||
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
|
||||||
|
@BatchOwner(Watcher.MaJinTao)
|
||||||
|
class CommunityHouseSampleBatch extends BaseBatch {
|
||||||
|
|
||||||
|
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||||
|
|
||||||
|
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||||
|
|
||||||
|
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
|
||||||
|
|
||||||
|
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017)
|
||||||
|
|
||||||
|
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||||
|
|
||||||
|
static void main(String[] args) {
|
||||||
|
runBatch(new CommunityHouseSampleBatch(), args)
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHouseCase(Object district_stat, Map<String, List> houseMap) {
|
||||||
|
|
||||||
|
// 查本小区的房源
|
||||||
|
def houseQuery = new BasicDBObject()
|
||||||
|
//houseQuery.put("city", "深圳")
|
||||||
|
|
||||||
|
//houseQuery.put("region", "福田")
|
||||||
|
|
||||||
|
//houseQuery.put("district_name", "中泰燕南名庭")
|
||||||
|
|
||||||
|
houseQuery.put("cat", "sell")
|
||||||
|
|
||||||
|
houseQuery.put("city", district_stat.city)
|
||||||
|
|
||||||
|
houseQuery.put("region", district_stat.region)
|
||||||
|
|
||||||
|
houseQuery.put("district_name", district_stat.district_name)
|
||||||
|
|
||||||
|
def houseListAll = SEABIRD.find(houseQuery).toList()
|
||||||
|
|
||||||
|
// 将均价符合的房源放入结果集
|
||||||
|
def houseList = houseListAll.findAll() { housecase ->
|
||||||
|
def totalPrice = housecase.total_price
|
||||||
|
def houseArea = housecase.area
|
||||||
|
def houseAvgPrice = totalPrice / houseArea
|
||||||
|
def avgPrice = district_stat.avg_price
|
||||||
|
if (houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2) {
|
||||||
|
return true
|
||||||
|
} else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// 将样本数据按面积段分,200以下每10为一段,200以上每30为一段
|
||||||
|
// 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20
|
||||||
|
|
||||||
|
for (def housecase : houseList) {
|
||||||
|
if (housecase.area <= 200) {
|
||||||
|
def area = housecase.area.toInteger()
|
||||||
|
|
||||||
|
def key = "g" + (area / 10).toInteger() * 10
|
||||||
|
// println(key)
|
||||||
|
// 获取该面积段的房源列表
|
||||||
|
def gList = houseMap.get(key)
|
||||||
|
|
||||||
|
// 如果为空,说明该面积段没有房源,新建一个list,放入
|
||||||
|
if (gList == null || gList.size() == 0) {
|
||||||
|
def gHouseCaseList = []
|
||||||
|
gHouseCaseList.add(housecase)
|
||||||
|
houseMap.put(key, gHouseCaseList)
|
||||||
|
} else {
|
||||||
|
// 如果该面积段已经有10个房源,就不再放入
|
||||||
|
if (gList.size() < 10) {
|
||||||
|
// 如果不为空,说明该面积段已经有房源,直接放入
|
||||||
|
gList.add(housecase)
|
||||||
|
houseMap.put(key, gList)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
def area = housecase.area.toInteger()
|
||||||
|
def startPoint = 200
|
||||||
|
def segmentSize = 30
|
||||||
|
def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger()
|
||||||
|
def key = "g" + rangeStart
|
||||||
|
// println(key)
|
||||||
|
// 获取该面积段的房源列表
|
||||||
|
def gList = houseMap.get(key)
|
||||||
|
|
||||||
|
// 如果为空,说明该面积段没有房源,新建一个list,放入
|
||||||
|
if (gList == null || gList.size() == 0) {
|
||||||
|
def gHouseCaseList = []
|
||||||
|
gHouseCaseList.add(housecase)
|
||||||
|
houseMap.put(key, gHouseCaseList)
|
||||||
|
} else {
|
||||||
|
// 如果该面积段已经有10个房源,就不再放入
|
||||||
|
if (gList.size() < 10) {
|
||||||
|
gList.add(housecase)
|
||||||
|
houseMap.put(key, gList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
|
|
||||||
|
|
||||||
|
def count = 0
|
||||||
|
|
||||||
|
|
||||||
|
def cityList = CommonUtils.findAllCity()
|
||||||
|
|
||||||
|
// 测试用,先只取一条
|
||||||
|
// cityList = cityList.subList(0, 1)
|
||||||
|
|
||||||
|
print(cityList.size())
|
||||||
|
print(cityList.toString())
|
||||||
|
// cityList.each { city ->
|
||||||
|
def city = cityList.get(0)
|
||||||
|
print(city)
|
||||||
|
def documents = []
|
||||||
|
//查询该城市的所有小区
|
||||||
|
List<BasicDBObject> districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||||
|
|
||||||
|
//测试用,先只取5个小区
|
||||||
|
// districts = districts.subList(0, 2)
|
||||||
|
|
||||||
|
// 查询所有的小区均价
|
||||||
|
|
||||||
|
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||||
|
def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend)
|
||||||
|
for (def district : districts) {
|
||||||
|
Map<String, List> houseMap = [:]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name)
|
||||||
|
// 拿到小区对应的均价之后,去查房源
|
||||||
|
if (district_stat != null) {
|
||||||
|
findHouseCase(district_stat, houseMap)
|
||||||
|
|
||||||
|
}
|
||||||
|
// 没有拿到小区对应的均价,找小区周边的三十个小区
|
||||||
|
|
||||||
|
def aroundDistrictListAll = DbUtils.findAroundDistrictByRadius(district, 3000)
|
||||||
|
def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll
|
||||||
|
|
||||||
|
|
||||||
|
// 遍历周边小区,找到均价符合的房源
|
||||||
|
|
||||||
|
aroundDistrictList.each { aroundDistrict ->
|
||||||
|
def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name)
|
||||||
|
// 拿到小区对应的均价之后,去查房源
|
||||||
|
if (aroundDistrict_stat != null) {
|
||||||
|
findHouseCase(aroundDistrict_stat, houseMap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print(houseMap)
|
||||||
|
|
||||||
|
// 创建一个新的文档
|
||||||
|
def document = new BasicDBObject()
|
||||||
|
|
||||||
|
// 将 houseMap 和 communityStr 放入文档
|
||||||
|
document.put("city", district.city)
|
||||||
|
document.put("region", district.region)
|
||||||
|
document.put("district_name", district.name)
|
||||||
|
document.put("house_case", houseMap)
|
||||||
|
|
||||||
|
// 将文档插入到 MongoDB 中
|
||||||
|
// HOUSE_CASE.in(document)
|
||||||
|
|
||||||
|
documents.add(document)
|
||||||
|
count++
|
||||||
|
if (count % 10 == 0) {
|
||||||
|
print("已处理小区数----------------------------------------------------" + count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HOUSE_CASE.insert(documents)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user