diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index cd1282a81..4d279deb8 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -530,7 +530,13 @@ + + + + + + - + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/CommunityHouseSampleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/CommunityHouseSampleBatch.groovy index 41494caf2..7e007ed94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/CommunityHouseSampleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/CommunityHouseSampleBatch.groovy @@ -1,13 +1,350 @@ -package com.fangjia.script.sync - +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection @BatchOwner(Watcher.MaJinTao) -class CommunityHouseSampleBatch extends BaseBatch{ +class CommunityHouseSampleBatch extends BaseBatch { + + DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017) + DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017) + DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird") + + static void main(String[] args) { + runBatch(new CommunityHouseSampleBatch(), args) + } + + def findHouseCase(Object district_stat, Map houseMap) { + // 查本小区的房源 + def houseQuery = new BasicDBObject() + houseQuery.put("cat", "sell") + houseQuery.put("city", district_stat.city) + houseQuery.put("region", district_stat.region) + houseQuery.put("district_name", district_stat.district_name) + def houseListAll = SEABIRD.find(houseQuery).toList() + + // 将均价符合的房源放入结果集 + def houseList = houseListAll.findAll() { housecase -> + def totalPrice = housecase.total_price + def houseArea = housecase.area + def houseAvgPrice = totalPrice / houseArea + def avgPrice = district_stat.avg_price + houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2 + } + + // 将样本数据按面积段分,200以下每10为一段,200以上每30为一段 + // 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20 + for (def housecase : houseList) { + if (housecase.area <= 200) { + def area = housecase.area.toInteger() + def key = "g" + (area / 10).toInteger() * 10 + def gList = houseMap.get(key) + + if (gList == null || gList.size() == 0) { + def gHouseCaseList = [] + gHouseCaseList.add(housecase) + houseMap.put(key, gHouseCaseList) + } else if (gList.size() < 10) { + gList.add(housecase) + houseMap.put(key, gList) + } + } else { + def area = housecase.area.toInteger() + def startPoint = 200 + def segmentSize = 30 + def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger() + def key = "g" + rangeStart + def gList = houseMap.get(key) + + if (gList == null || gList.size() == 0) { + def gHouseCaseList = [] + gHouseCaseList.add(housecase) + houseMap.put(key, gHouseCaseList) + } else if (gList.size() < 10) { + gList.add(housecase) + houseMap.put(key, gList) + } + } + } + } + + @Override + void process(Object args) { + ExecutorService executorService = Executors.newFixedThreadPool(5) + + try { + def cityList = CommonUtils.findAllCity() + +// cityList = cityList.subList(0,150) + cityList.each { city -> + executorService.submit({ + def count = 0 + def documents = [] + def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend) + List districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) + + // 测试,只选一个 +// districts = districts.subList(0, 1) + + districts.each { district -> + Map houseMap = [:] + def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name) + + if (district_stat != null) { + findHouseCase(district_stat, houseMap) + } + if (houseMap.size() == 0) { + def aroundDistrictList = DbUtils.findAroundDistrictByRadius(district, 3000) +// def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll + aroundDistrictList.each { aroundDistrict -> + def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name) + if (aroundDistrict_stat != null) { + findHouseCase(aroundDistrict_stat, houseMap) + } + } + } + + def document = new BasicDBObject() + document.put("city", district.city) + document.put("region", district.region) + document.put("district_name", district.name) + document.put("house_case", houseMap) + + documents.add(document) + count++ + if (count % 10 == 0) { + println(city + "已处理小区数----------------------------------------------------" + count) + } + } + HOUSE_CASE.insert(documents) + } as Runnable) + } + } finally { + executorService.shutdown() + executorService.awaitTermination(Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS) + + } + } +} + + +/* +package com.fangjia.script.sync + +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + + +@BatchOwner(Watcher.MaJinTao) +class CommunityHouseSampleBatch extends BaseBatch { + + DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017) + + DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + + DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017) + +// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017) + + DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird") + + static void main(String[] args) { + runBatch(new CommunityHouseSampleBatch(), args) + } + + def findHouseCase(Object district_stat, Map houseMap) { + + // 查本小区的房源 + def houseQuery = new BasicDBObject() + //houseQuery.put("city", "深圳") + + //houseQuery.put("region", "福田") + + //houseQuery.put("district_name", "中泰燕南名庭") + + houseQuery.put("cat", "sell") + + houseQuery.put("city", district_stat.city) + + houseQuery.put("region", district_stat.region) + + houseQuery.put("district_name", district_stat.district_name) + + def houseListAll = SEABIRD.find(houseQuery).toList() + + // 将均价符合的房源放入结果集 + def houseList = houseListAll.findAll() { housecase -> + def totalPrice = housecase.total_price + def houseArea = housecase.area + def houseAvgPrice = totalPrice / houseArea + def avgPrice = district_stat.avg_price + if (houseAvgPrice >= avgPrice * 0.8 && houseAvgPrice <= avgPrice * 1.2) { + return true + } else { + return false + } + } + + + // 将样本数据按面积段分,200以下每10为一段,200以上每30为一段 + // 分段的字段规则 area: 23.4 对应[20,30] 标记为 g20 + + for (def housecase : houseList) { + if (housecase.area <= 200) { + def area = housecase.area.toInteger() + + def key = "g" + (area / 10).toInteger() * 10 +// println(key) + // 获取该面积段的房源列表 + def gList = houseMap.get(key) + + // 如果为空,说明该面积段没有房源,新建一个list,放入 + if (gList == null || gList.size() == 0) { + def gHouseCaseList = [] + gHouseCaseList.add(housecase) + houseMap.put(key, gHouseCaseList) + } else { + // 如果该面积段已经有10个房源,就不再放入 + if (gList.size() < 10) { + // 如果不为空,说明该面积段已经有房源,直接放入 + gList.add(housecase) + houseMap.put(key, gList) + + } + + } + } else { + def area = housecase.area.toInteger() + def startPoint = 200 + def segmentSize = 30 + def rangeStart = (startPoint + (Math.floor((area - startPoint) / segmentSize) * segmentSize)).toInteger() + def key = "g" + rangeStart +// println(key) + // 获取该面积段的房源列表 + def gList = houseMap.get(key) + + // 如果为空,说明该面积段没有房源,新建一个list,放入 + if (gList == null || gList.size() == 0) { + def gHouseCaseList = [] + gHouseCaseList.add(housecase) + houseMap.put(key, gHouseCaseList) + } else { + // 如果该面积段已经有10个房源,就不再放入 + if (gList.size() < 10) { + gList.add(housecase) + houseMap.put(key, gList) + } + } + } + + + } + } + @Override void process(Object args) { + + def count = 0 + + + def cityList = CommonUtils.findAllCity() + + // 测试用,先只取一条 +// cityList = cityList.subList(0, 1) + + print(cityList.size()) + print(cityList.toString()) +// cityList.each { city -> + def city = cityList.get(0) + print(city) + def documents = [] + //查询该城市的所有小区 + List districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) + + //测试用,先只取5个小区 +// districts = districts.subList(0, 2) + + // 查询所有的小区均价 + + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def districtPriceMap = CommonDatautils.queryAllDistrictPrice(city, weekend) + for (def district : districts) { + Map houseMap = [:] + + + + def district_stat = districtPriceMap.get(district.city + "##" + district.region + "##" + district.name) + // 拿到小区对应的均价之后,去查房源 + if (district_stat != null) { + findHouseCase(district_stat, houseMap) + + } + // 没有拿到小区对应的均价,找小区周边的三十个小区 + + def aroundDistrictListAll = DbUtils.findAroundDistrictByRadius(district, 3000) + def aroundDistrictList = (aroundDistrictListAll.size() >= 50) ? aroundDistrictListAll.subList(0, 50) : aroundDistrictListAll + + + // 遍历周边小区,找到均价符合的房源 + + aroundDistrictList.each { aroundDistrict -> + def aroundDistrict_stat = districtPriceMap.get(aroundDistrict.city + "##" + aroundDistrict.region + "##" + aroundDistrict.name) + // 拿到小区对应的均价之后,去查房源 + if (aroundDistrict_stat != null) { + findHouseCase(aroundDistrict_stat, houseMap) + } + } + print(houseMap) + + // 创建一个新的文档 + def document = new BasicDBObject() + + // 将 houseMap 和 communityStr 放入文档 + document.put("city", district.city) + document.put("region", district.region) + document.put("district_name", district.name) + document.put("house_case", houseMap) + + // 将文档插入到 MongoDB 中 +// HOUSE_CASE.in(document) + + documents.add(document) + count++ + if (count % 10 == 0) { + print("已处理小区数----------------------------------------------------" + count) + } + } + HOUSE_CASE.insert(documents) + } + +// } } + +*/ + + + diff --git a/sdklog/sdkclient_CommunicateLog_info b/sdklog/sdkclient_CommunicateLog_info new file mode 100644 index 000000000..e69de29bb diff --git a/sdklog/sdkclient_EMAYSDK_info b/sdklog/sdkclient_EMAYSDK_info new file mode 100644 index 000000000..e69de29bb