Merge branch 'script_gj' into 'master'
数据量太多按城市加载 See merge request !903
This commit is contained in:
+14
@@ -0,0 +1,14 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
|
||||
if(d['G'] && d['G'] instanceof String){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
if(obj){
|
||||
d['H'] = obj.name
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GreenBuild {
|
||||
|
||||
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new GreenBuild()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAll()
|
||||
def rets = match(datas)
|
||||
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
|
||||
}
|
||||
|
||||
def match(datas){
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
|
||||
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
data.match_id = resultMap.get("id").toString()
|
||||
data.match_city = resultMap.get("city")
|
||||
data.match_district = resultMap.get("district")
|
||||
data.match_name = resultMap.get("name")
|
||||
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
|
||||
}else{
|
||||
rets << [data._id.toString(), data.city, data.district, data.name]
|
||||
}
|
||||
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await();
|
||||
return rets
|
||||
}
|
||||
|
||||
def findAll(){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
|
||||
}
|
||||
|
||||
}
|
||||
+4
-3
@@ -94,10 +94,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def endDate = curMonth+"19" //正式运行时curMonth要改
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
def houseIdsfilter = null;//findAllHouseIdsByCity()
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
houseIdsfilter = findAllHouseIdsByCity(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
@@ -919,11 +920,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
def findAllHouseIdsByCity(){
|
||||
def findAllHouseIdsByCity(city){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 800000000, 0.000001);
|
||||
// def houseIds = new HashSet()
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_nm='${city}'",{it->
|
||||
filter.put(it.gujia_sell_id)
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user