This commit is contained in:
2022-07-20 16:44:35 +08:00
parent 20940fa2c0
commit 62d9fd7940
2 changed files with 413 additions and 0 deletions
@@ -0,0 +1,138 @@
package com.fangjia.script.chongqing
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class PocSampleTest {
static main(args) {
def task = new PocSampleTest()
task.run()
}
def run(){
// match("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx")
// addPrice("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx")
// addArroundPrice("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx")
// addRentRate("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx")
// matchResult("D:\\数据\\美团\\抽样_1000_结果.xlsx")
existsDist("D:\\数据\\美团\\抽样_1000_结果.xlsx")
}
def existsDist(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['P']){
def dist = CommonDatautils.findDistrictById(row['P'])
if(!dist){
row['S'] = "不存在"
}
}
})
}
def matchResult(path){
def datas = ExcelUtils.read("D:\\数据\\美团\\美团金融小区测试list-所有数据-优化租金评级.xlsx", 0 ,1)
def map = [:]
for(def r : datas){
map.put(r['D'], r)
}
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A']){
def d = map.get(row['A'])
if(d){
def obj = CommonDatautils.findDistrict(d['F'],d['G'],d['H'],"district")
if(obj){
row['P'] = obj._id.toString()
row['Q'] = d['H']
row['R'] = d['I']
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['G'] && row['D']){
def city = row['D']
def region = row['E']
if(region && region=='NULL'){
region = row['F']
}
def address = row['C']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, row['B'], null, true, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['H'] = jsonObject.get("name")
row['G'] = jsonObject.get("id")
row['I'] = jsonObject.get("address")
row['L'] = "地图匹配(名称当地址)"
row['M'] = jsonObject.get("propertyType")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['G']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// row['J'] = priceObj.avg_price
// }
def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(dist.city, dist.region, dist.name)
if(sqmetreRentPrice){
row['K'] = sqmetreRentPrice
}
}
}
})
}
def addArroundPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['J']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['D'], row['C'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['J'] = priceObj.unitPrice
row['N'] = "周边均价"
}
}
});
}
def addRentRate(path){
def citys = ["东莞","苏州","郑州","长沙"]
def radioMap = [:]
for(def city : citys) {
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
radioMap.put(city, avg_radio)
}
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['D']
if (city && row['J'] && !row['K']) {
city = city.replace("市","")
def avg_rent_radio = radioMap.get(city)
if(avg_rent_radio){
row['K'] = (int)(row['J']/avg_rent_radio)
row['O'] = "租金推算"
}
}
})
}
}
@@ -0,0 +1,275 @@
package com.fangjia.script.gj.test
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
//def q = new BasicDBObject()
//q.put("city","重庆")
//q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")))
//def dbcursor = DEAL_PRICE_COLL.find(q)
//def map = [:]
//for(def d : dbcursor){
// def list = map.get(d.region)
// if(!list){
// list = []
// map.put(d.region, list)
// }
// def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd")
// if(!list.contains(key)){
// list << key
// }
//}
//def q = new BasicDBObject()
//q.put("status","已成交")
//q.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")))
//def dbcursor = paimai43_COLL.find(q)
//def map = [:]
//for(def d : dbcursor){
// def list = map.get(d.region)
// if(!list){
// list = []
// map.put(d.region, list)
// }
// if(!list.contains(d.title)){
// list << d.title
// }
//}
//ExcelUtils.modify("D:\\数据\\朔州房源.xlsx",{r->
// def dist = CommonDatautils.findDistrict(r['A'],r['B'], r['C'], "district")
// if(dist){
// r['E'] = "名称完全一致"
// }
//
//})
def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"]
def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(rMap.city, rMap.region, "sell", 20210701,rMap.name, rMap.mapping_alias)
println totalCount.get() +" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
// if(!ret){
// ret = findThirdParty(rMap.city, rMap.region, rMap.name, AVERAGE_STATISTICS)
// }
if(ret && ret.size()<=3){
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
retMap.put(key, ret.size())
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("D:\\数据\\房源小于4个以下的小区.xlsx", datas, "数量")
//def rets = findHousesByCity("朔州", "sell", 20210701)
//def map = [:]
//for(def r : rets){
// def key = r.city+"#"+r.region+"#"+r.district_name
// def count = map.get(key)
// if(!count){
// count = 1
// map.put(key, count)
// }else{
// map.put(key, count+1)
// }
//}
//
//def datas = []
//datas << ["城市","区域","名称","数量"]
//retMap.each {k,count->
// def arrs = k.split("#")
// datas << [arrs[0], arrs[1], arrs[2], count]
//}
//ExcelUtils.write("D:\\数据\\朔州匹配房源1.xlsx", datas, "数量")
//ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆数据统计.xlsx",{d->
// def count = retMap.get(d['A'])
// if(count){
// def pre = count.toInteger()*100/d['B'].toString().toDouble().toInteger()
// pre = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, pre)
// d['E'] = pre
// }
// if(d['A'] && !d['H']){
// def prices = CommonDatautils.queryCityRegionPrices("重庆", d['A'], '2022-01-01')
// if(prices){
// def avg_prices = prices.sum()/prices.size()
// d['H'] = avg_prices.toInteger()
// }
// def prices = CommonDatautils.queryDistrictPrices("重庆", d['A'], '2022-01-01')
// if(prices){
// def avg_prices = prices.sum()/prices.size()
// d['H'] = avg_prices.toInteger()
// }
// }
//})
//ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d->
// def rets = findHousesByCityAndRoot(d['A'], "sell", 20210701, "anjuke.com")
// d['R'] = rets.size()
// println d['A'] +" "+rets.size()
//
//})
def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058]
ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d->
def count = countMap.get(d['A'])
d['R'] = count
})
println "完成"
def findThirdParty(city, region, name, coll){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202107))
def dbcursor = coll.find(q)
def rets = []
for(def d : dbcursor){
if(d && d.size()>9){
println d
rets << d
}
}
return rets
}
def findHouses(def city, def region, def cat, def startDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses;
}
def findHousesByCity(def city, def cat, def startDate){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses;
}
def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(QueryBuilders.termQuery("root", root))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses;
}
//def map = [:]
//for(def d : dists){
// def list = map.get(d.region)
// if(!list){
// list = []
// map.put(d.region, list)
// }
// list << d
//}
//
//def rows = []
//rows << ["区域","小区量", "别名量", "地址别名量"]
//map.each{region, list->
// def alias_count = 0
// def addr_alias_count = 0
// for(def r : list){
// if(r.mapping_alias && r.mapping_alias.size()>0){
// alias_count ++
// }
// if(r.addr_alias && r.addr_alias.size()>0){
// addr_alias_count++
// }
// }
//
// rows << [region, list.size(), alias_count, addr_alias_count]
//}
//
//ExcelUtils.write("D:\\数据\\重庆农商行\\重庆数据统计.xlsx", rows, "楼盘统计")