From 62d9fd794055cdb845eef1f0d59c7f7da6b414aa Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Jul 2022 16:44:35 +0800 Subject: [PATCH] update --- .../script/chongqing/PocSampleTest.groovy | 138 +++++++++ .../script/gj/test/DataStatTest.groovy | 275 ++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy new file mode 100644 index 000000000..509e35e88 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy @@ -0,0 +1,138 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class PocSampleTest { + + static main(args) { + def task = new PocSampleTest() + task.run() + } + + def run(){ +// match("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") +// addPrice("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") +// addArroundPrice("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") +// addRentRate("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") + +// matchResult("D:\\数据\\美团\\抽样_1000_结果.xlsx") + existsDist("D:\\数据\\美团\\抽样_1000_结果.xlsx") + } + + def existsDist(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['P']){ + def dist = CommonDatautils.findDistrictById(row['P']) + if(!dist){ + row['S'] = "不存在" + } + } + }) + } + + def matchResult(path){ + def datas = ExcelUtils.read("D:\\数据\\美团\\美团金融小区测试list-所有数据-优化租金评级.xlsx", 0 ,1) + def map = [:] + for(def r : datas){ + map.put(r['D'], r) + } + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def d = map.get(row['A']) + if(d){ + def obj = CommonDatautils.findDistrict(d['F'],d['G'],d['H'],"district") + if(obj){ + row['P'] = obj._id.toString() + row['Q'] = d['H'] + row['R'] = d['I'] + } + } + } + }) + + } + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['G'] && row['D']){ + def city = row['D'] + def region = row['E'] + if(region && region=='NULL'){ + region = row['F'] + } + def address = row['C'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, row['B'], null, true, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['H'] = jsonObject.get("name") + row['G'] = jsonObject.get("id") + row['I'] = jsonObject.get("address") + row['L'] = "地图匹配(名称当地址)" + row['M'] = jsonObject.get("propertyType") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['G'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// row['J'] = priceObj.avg_price +// } + + def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(dist.city, dist.region, dist.name) + if(sqmetreRentPrice){ + row['K'] = sqmetreRentPrice + } + } + } + + + }) + } + + def addArroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['J']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['D'], row['C']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['J'] = priceObj.unitPrice + row['N'] = "周边均价" + } + } + }); + } + + def addRentRate(path){ + def citys = ["东莞","苏州","郑州","长沙"] + def radioMap = [:] + for(def city : citys) { + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) + radioMap.put(city, avg_radio) + } + + ExcelUtils.multiThreadingModify(path, { row -> + def city = row['D'] + if (city && row['J'] && !row['K']) { + city = city.replace("市","") + def avg_rent_radio = radioMap.get(city) + if(avg_rent_radio){ + row['K'] = (int)(row['J']/avg_rent_radio) + row['O'] = "租金推算" + } + } + }) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy new file mode 100644 index 000000000..fdc7e3197 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy @@ -0,0 +1,275 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) +DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) +DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + +DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43") + +//def q = new BasicDBObject() +//q.put("city","重庆") +//q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))) +//def dbcursor = DEAL_PRICE_COLL.find(q) +//def map = [:] +//for(def d : dbcursor){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } +// def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd") +// if(!list.contains(key)){ +// list << key +// } +//} + + +//def q = new BasicDBObject() +//q.put("status","已成交") +//q.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))) +//def dbcursor = paimai43_COLL.find(q) +//def map = [:] +//for(def d : dbcursor){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } +// if(!list.contains(d.title)){ +// list << d.title +// } +//} + + +//ExcelUtils.modify("D:\\数据\\朔州房源.xlsx",{r-> +// def dist = CommonDatautils.findDistrict(r['A'],r['B'], r['C'], "district") +// if(dist){ +// r['E'] = "名称完全一致" +// } +// +//}) + +def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"] + +def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"]) + +def totalCount = new AtomicLong() +final CountDownLatch latch = new CountDownLatch(districts.size()) +println "小区量:"+districts.size() + +def retMap = new ConcurrentHashMap() +for(def i=0;i +// def arrs = k.split("#") +// datas << [arrs[0], arrs[1], arrs[2], count] +//} +//ExcelUtils.write("D:\\数据\\朔州匹配房源1.xlsx", datas, "数量") +//ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆数据统计.xlsx",{d-> +// def count = retMap.get(d['A']) +// if(count){ +// def pre = count.toInteger()*100/d['B'].toString().toDouble().toInteger() +// pre = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, pre) +// d['E'] = pre +// } +// if(d['A'] && !d['H']){ +// def prices = CommonDatautils.queryCityRegionPrices("重庆", d['A'], '2022-01-01') +// if(prices){ +// def avg_prices = prices.sum()/prices.size() +// d['H'] = avg_prices.toInteger() +// } +// def prices = CommonDatautils.queryDistrictPrices("重庆", d['A'], '2022-01-01') +// if(prices){ +// def avg_prices = prices.sum()/prices.size() +// d['H'] = avg_prices.toInteger() +// } +// } +//}) + + +//ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d-> +// def rets = findHousesByCityAndRoot(d['A'], "sell", 20210701, "anjuke.com") +// d['R'] = rets.size() +// println d['A'] +" "+rets.size() +// +//}) + +def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058] +ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d-> + def count = countMap.get(d['A']) + d['R'] = count +}) + +println "完成" + + +def findThirdParty(city, region, name, coll){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', 202107)) + def dbcursor = coll.find(q) + def rets = [] + for(def d : dbcursor){ + if(d && d.size()>9){ + println d + rets << d + } + } + return rets +} + + +def findHouses(def city, def region, def cat, def startDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; +} + + +def findHousesByCity(def city, def cat, def startDate){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; +} + +def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range).must(QueryBuilders.termQuery("root", root)) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; +} + +//def map = [:] +//for(def d : dists){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } +// list << d +//} +// +//def rows = [] +//rows << ["区域","小区量", "别名量", "地址别名量"] +//map.each{region, list-> +// def alias_count = 0 +// def addr_alias_count = 0 +// for(def r : list){ +// if(r.mapping_alias && r.mapping_alias.size()>0){ +// alias_count ++ +// } +// if(r.addr_alias && r.addr_alias.size()>0){ +// addr_alias_count++ +// } +// } +// +// rows << [region, list.size(), alias_count, addr_alias_count] +//} +// +//ExcelUtils.write("D:\\数据\\重庆农商行\\重庆数据统计.xlsx", rows, "楼盘统计")