数据量太多按城市加载

This commit is contained in:
2024-08-12 11:13:32 +08:00
parent d270779225
commit ece0965637
3 changed files with 97 additions and 3 deletions
@@ -0,0 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
if(d['G'] && d['G'] instanceof String){
def obj = CommonDatautils.findDistrictById(d['G'])
if(obj){
d['H'] = obj.name
}
}
})
@@ -0,0 +1,79 @@
package com.fangjia.script.gj
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class GreenBuild {
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new GreenBuild()
task.run()
}
def run(){
def datas = findAll()
def rets = match(datas)
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
}
def match(datas){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
data.match_id = resultMap.get("id").toString()
data.match_city = resultMap.get("city")
data.match_district = resultMap.get("district")
data.match_name = resultMap.get("name")
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
}else{
rets << [data._id.toString(), data.city, data.district, data.name]
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await();
return rets
}
def findAll(){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
}
}
@@ -94,10 +94,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
def houseIdsfilter = null;//findAllHouseIdsByCity()
for(def city : citys){
index++
houseIdsfilter = findAllHouseIdsByCity(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
@@ -919,11 +920,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(){
def findAllHouseIdsByCity(city){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 800000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_nm='${city}'",{it->
filter.put(it.gujia_sell_id)
});