按小区存房源

This commit is contained in:
mjt
2023-12-29 09:35:52 +08:00
parent 5a7b65c704
commit 8dd7ad650a
4 changed files with 347 additions and 4 deletions
+7 -1
View File
@@ -530,7 +530,13 @@
</exclusion> </exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
<!-- <version>0.0.1-SNAPSHOT</version>-->
<!-- <scope>compile</scope>-->
<!-- </dependency>-->
</dependencies> </dependencies>
</project> </project>
@@ -1,13 +1,350 @@
package com.fangjia.script.sync import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.MaJinTao) @BatchOwner(Watcher.MaJinTao)
class CommunityHouseSampleBatch extends BaseBatch{ class CommunityHouseSampleBatch extends BaseBatch {
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
static void main(String[] args) {
runBatch(new CommunityHouseSampleBatch(), args)
}
def findHouseCase(Object district_stat, Map<String, List> houseMap) {
// 查本小区的房源
def houseQuery = new BasicDBObject()
houseQuery.put("cat", "sell")
houseQuery.put("city", district_stat.city)
houseQuery.put("region", district_stat.region)
houseQuery.put("district_name", district_stat.district_name)
def houseListAll = SEABIRD.find(houseQuery).toList()
// 将均价符合的房源放入结果集
def houseList = houseListAll.findAll() { housecase ->
def totalPrice = housecase.total_price
def houseArea = housecase.area
def houseAvgPrice = totalPrice / houseArea
def avgPrice = district_stat.avg_price
houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2
}
// 将样本数据按面积段分,200以下每10为一段,200以上每30为一段
// 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20
for (def housecase : houseList) {
if (housecase.area <= 200) {
def area = housecase.area.toInteger()
def key = "g" + (area / 10).toInteger() * 10
def gList = houseMap.get(key)
if (gList == null || gList.size() == 0) {
def gHouseCaseList = []
gHouseCaseList.add(housecase)
houseMap.put(key, gHouseCaseList)
} else if (gList.size() < 10) {
gList.add(housecase)
houseMap.put(key, gList)
}
} else {
def area = housecase.area.toInteger()
def startPoint = 200
def segmentSize = 30
def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger()
def key = "g" + rangeStart
def gList = houseMap.get(key)
if (gList == null || gList.size() == 0) {
def gHouseCaseList = []
gHouseCaseList.add(housecase)
houseMap.put(key, gHouseCaseList)
} else if (gList.size() < 10) {
gList.add(housecase)
houseMap.put(key, gList)
}
}
}
}
@Override
void process(Object args) {
ExecutorService executorService = Executors.newFixedThreadPool(5)
try {
def cityList = CommonUtils.findAllCity()
// cityList = cityList.subList(0,150)
cityList.each { city ->
executorService.submit({
def count = 0
def documents = []
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend)
List<BasicDBObject> districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
// 测试,只选一个
// districts = districts.subList(0, 1)
districts.each { district ->
Map<String, List> houseMap = [:]
def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name)
if (district_stat != null) {
findHouseCase(district_stat, houseMap)
}
if (houseMap.size() == 0) {
def aroundDistrictList = DbUtils.findAroundDistrictByRadius(district, 3000)
// def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll
aroundDistrictList.each { aroundDistrict ->
def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name)
if (aroundDistrict_stat != null) {
findHouseCase(aroundDistrict_stat, houseMap)
}
}
}
def document = new BasicDBObject()
document.put("city", district.city)
document.put("region", district.region)
document.put("district_name", district.name)
document.put("house_case", houseMap)
documents.add(document)
count++
if (count % 10 == 0) {
println(city + "已处理小区数----------------------------------------------------" + count)
}
}
HOUSE_CASE.insert(documents)
} as Runnable)
}
} finally {
executorService.shutdown()
executorService.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS)
}
}
}
/*
package com.fangjia.script.sync
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.MaJinTao)
class CommunityHouseSampleBatch extends BaseBatch {
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017)
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
static void main(String[] args) {
runBatch(new CommunityHouseSampleBatch(), args)
}
def findHouseCase(Object district_stat, Map<String, List> houseMap) {
// 查本小区的房源
def houseQuery = new BasicDBObject()
//houseQuery.put("city", "深圳")
//houseQuery.put("region", "福田")
//houseQuery.put("district_name", "中泰燕南名庭")
houseQuery.put("cat", "sell")
houseQuery.put("city", district_stat.city)
houseQuery.put("region", district_stat.region)
houseQuery.put("district_name", district_stat.district_name)
def houseListAll = SEABIRD.find(houseQuery).toList()
// 将均价符合的房源放入结果集
def houseList = houseListAll.findAll() { housecase ->
def totalPrice = housecase.total_price
def houseArea = housecase.area
def houseAvgPrice = totalPrice / houseArea
def avgPrice = district_stat.avg_price
if (houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2) {
return true
} else {
return false
}
}
// 将样本数据按面积段分,200以下每10为一段,200以上每30为一段
// 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20
for (def housecase : houseList) {
if (housecase.area <= 200) {
def area = housecase.area.toInteger()
def key = "g" + (area / 10).toInteger() * 10
// println(key)
// 获取该面积段的房源列表
def gList = houseMap.get(key)
// 如果为空,说明该面积段没有房源,新建一个list,放入
if (gList == null || gList.size() == 0) {
def gHouseCaseList = []
gHouseCaseList.add(housecase)
houseMap.put(key, gHouseCaseList)
} else {
// 如果该面积段已经有10个房源,就不再放入
if (gList.size() < 10) {
// 如果不为空,说明该面积段已经有房源,直接放入
gList.add(housecase)
houseMap.put(key, gList)
}
}
} else {
def area = housecase.area.toInteger()
def startPoint = 200
def segmentSize = 30
def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger()
def key = "g" + rangeStart
// println(key)
// 获取该面积段的房源列表
def gList = houseMap.get(key)
// 如果为空,说明该面积段没有房源,新建一个list,放入
if (gList == null || gList.size() == 0) {
def gHouseCaseList = []
gHouseCaseList.add(housecase)
houseMap.put(key, gHouseCaseList)
} else {
// 如果该面积段已经有10个房源,就不再放入
if (gList.size() < 10) {
gList.add(housecase)
houseMap.put(key, gList)
}
}
}
}
}
@Override @Override
void process(Object args) { void process(Object args) {
def count = 0
def cityList = CommonUtils.findAllCity()
// 测试用,先只取一条
// cityList = cityList.subList(0, 1)
print(cityList.size())
print(cityList.toString())
// cityList.each { city ->
def city = cityList.get(0)
print(city)
def documents = []
//查询该城市的所有小区
List<BasicDBObject> districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//测试用,先只取5个小区
// districts = districts.subList(0, 2)
// 查询所有的小区均价
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend)
for (def district : districts) {
Map<String, List> houseMap = [:]
def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name)
// 拿到小区对应的均价之后,去查房源
if (district_stat != null) {
findHouseCase(district_stat, houseMap)
}
// 没有拿到小区对应的均价,找小区周边的三十个小区
def aroundDistrictListAll = DbUtils.findAroundDistrictByRadius(district, 3000)
def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll
// 遍历周边小区,找到均价符合的房源
aroundDistrictList.each { aroundDistrict ->
def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name)
// 拿到小区对应的均价之后,去查房源
if (aroundDistrict_stat != null) {
findHouseCase(aroundDistrict_stat, houseMap)
}
}
print(houseMap)
// 创建一个新的文档
def document = new BasicDBObject()
// 将 houseMap 和 communityStr 放入文档
document.put("city", district.city)
document.put("region", district.region)
document.put("district_name", district.name)
document.put("house_case", houseMap)
// 将文档插入到 MongoDB 中
// HOUSE_CASE.in(document)
documents.add(document)
count++
if (count % 10 == 0) {
print("已处理小区数----------------------------------------------------" + count)
}
}
HOUSE_CASE.insert(documents)
} }
// }
} }
*/
View File