From e2a260b1a4ffd24f37e985c91e04d2d87103e9dd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Nov 2023 15:10:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E4=B8=8A=E6=B5=B7=E6=88=BF?= =?UTF-8?q?=E6=BA=90=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BatchAddPersonGuiPrice.groovy | 76 +++++ .../ccbevaluation/BigdataForsaleBatch.groovy | 3 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 7 +- .../district/DistrictFacilities2Batch.groovy | 97 ++++++ .../com/fangjia/script/gj/OpenApiUtils.groovy | 17 ++ .../script/gj/commStat/DataStatTest.groovy | 77 ++++- .../fangjia/script/gj/district/ApiTest.groovy | 155 +++++++++- .../script/gj/meishubao/MsbPocTest.groovy | 41 ++- .../gj/xiamen/SuggestionDataSynch.groovy | 286 ++++++++++++++++++ .../fangjia/script/util/OpenApiClient.groovy | 3 +- .../fangjia/script/util/ProvinceData.groovy | 43 +++ .../fangjia/script/wenyan/temp/QinLin.groovy | 3 +- .../fangjia/script/wenyan/temp/ZhuoXin.groovy | 6 +- 14 files changed, 784 insertions(+), 32 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictFacilities2Batch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy new file mode 100644 index 000000000..683c3048a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy @@ -0,0 +1,76 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 添加人工指导价,并重算价格 + * 小付的人工指导价不能覆盖 + */ +class BatchAddPersonGuiPrice { + + public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") + + static main(args) { + def task = new BatchAddPersonGuiPrice() + task.run() + } + + def run(){ + def path = "D:\\数据\\价格\\江苏剩下5城人工指导价录入.xlsx" + def datas = ExcelUtils.read(path, 0, 1) + for(def d : datas){ + def row = findGuidingPrice(d['B'], d['C'], d['D']) + if(!row){ + def obj = CommonDatautils.findDistrictById(d['A']) + save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2) + //发重算均价命令 + addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-10','yyyy-MM-dd')) + } + } + } + + + def findGuidingPrice(city, region, name) { + def q = new BasicDBObject(); + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("guiding_person" , "付贵生") + q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd"))) + def dbo = GUIDING_PRICE_COLL.findOne(q) + return dbo + } + + def save(city, region, name, price, estate_type){ + DBObject obj = new BasicDBObject(); + obj.put("city", city); + obj.put("region", region); + obj.put("name", name); + obj.put("guiding_date", new Date()-7); + obj.put("guiding_value", price.toString().toDouble().toInteger()); + obj.put("category", "district"); + obj.put("guiding_type", 0); + obj.put("c_date", new Date()); + obj.put("m_date", new Date()); + obj.put("estate_type" , estate_type); + obj.put("house_type" , 0) + obj.put("guiding_person" , "陈杰") + GUIDING_PRICE_COLL.save(obj); + } + + + def addPrice(city, region, name, estateType2,startDate, endDate){ + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + for(def weekend : weekends){ + CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index b1b8447f3..061b315af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,6 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ println "房源时间:"+startDate+" "+endDate def houseIdsfilter = findAllHouseIdsByCity() + citys = ["上海"] for(def city : citys){ index++ @@ -734,7 +735,7 @@ class BigdataForsaleBatch extends BaseBatch{ QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) - .must(date_range).must(area_range).must(date_range_mdate).must(QueryBuilders.termQuery("root", "lianjia.com")) + .must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com")) if(!alias){ alias = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index cf169cc97..acbd0b72c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-27") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-07") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 509887495..25102297b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -232,12 +232,11 @@ class CommonDatautils { public static def findRegionDistrictCount(def cat, def city, def region){ DBObject q = new BasicDBObject(); - q.put("visible", 0); + q.put("status", 0); q.put("cat", cat); q.put("city", city); q.put("region", region); - DSCursor cursor = getDataStoreConnection().find(q); - return cursor.count(); + return DbUtils.seaweed.count(q); } public static def findAllFiledDistrict(def cat, def city){ @@ -4002,7 +4001,7 @@ class CommonDatautils { def location = [:] location.lon = DataHandleUtils.getDoubleRound(6, point.x) location.lat = DataHandleUtils.getDoubleRound(6, point.y) - rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] + rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictFacilities2Batch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictFacilities2Batch.groovy new file mode 100644 index 000000000..2926e6551 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictFacilities2Batch.groovy @@ -0,0 +1,97 @@ +package com.fangjia.script.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +@BatchOwner(Watcher.GaoJuan) +class DistrictFacilities2Batch extends BaseBatch { + + public DBCollection onlineColl = MongoUtils.getCollectionByName("fangjia", "district_arround_poi", "mongo.db.fangjia.com") + + public static void main(String[] args) { + //小区周边数据更新 + this.runBatch(new DistrictFacilities2Batch(), args); + } + + + @Override + public void process(Object args) { + + def cities = CommonDatautils.findAllCity() + for (def city : cities) { + def ret = CommonDatautils.getAllDistricts("district", city) + def index = 0; + for (def dbo : ret) { + index++; + println index + "/" + ret.size() + " " + dbo.city + " " + dbo.name + + String region = dbo.get('region') + String districtName = dbo.get('name') + + if (!dbo.lat2 && !dbo.lng2) { + continue; + } + def poms = findPoms(dbo.lat2 / 10000000000.0, dbo.lng2 / 10000000000.0); + DBObject object = new BasicDBObject(); + object.put("hospital", unique(processType(poms.get('医院'), 1000, null, ['宠物', '动物', '口腔', '牙科', '美容', '齿科', '眼科', '号楼', '住院部', 'Pure', 'Tokushinkai', '假牙制作', '镶牙', '咪', '喵']))) + object.put("shop", processType(poms.get('超市'), 1000, ['便民商店', '便利店'], ['杂货店', '烟酒', '爱鑫', '超市便利店'])) + object.put("university", processType(poms.get('学校'), 1000, ['大学'], ['食堂', '办公室', '德威', '早教', '夕阳红', '公司'])) + object.put("middle_school", processType(poms.get('学校'), 1000, ['中学'], ['食堂', '办公室', '早教', '夕阳红', '公司'])) + object.put("primary_school", processType(poms.get('学校'), 1000, ['小学'], ['食堂', '办公室', '早教', '夕阳红', '公司'])) + object.put("kindergarten", processType(poms.get('学校'), 1000, ['幼儿园', '托儿所'], ['食堂', '亲子', '办公室', '夕阳红', '公司'])) + object.put("food_shop", processType(poms.get('餐饮'), 500, ['餐饮服务'])) + def market = []; + if (poms.get('商场')) { + market = market + poms.get('商场'); + } + if (poms.get('超市')) { + market = market + poms.get('超市'); + } + object.put("supermarket", processType(market, 1000, ['购物中心', "超市", "好又多"], ['公司', '村村寨寨', '伍缘折扣', '杂货店'])) + object.put("bank", processType(poms.get('银行'), 1000, ['银行'], ['自动提款机'])) + object.put("atm", processType(poms.get('ATM'), 1000, ['自动提款机', "atm", "ATM"])) + updateToDistrictFacilitys(city, region, districtName, object) + updateToDistrictFacilitysScore(city, region, districtName) + } + } + } + + + def findPoms(def lat, def lng) { + def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + + DBObject q = new BasicDBObject() + q.put('lat', new BasicDBObject(QueryOperators.LTE, map.maxLat).append(QueryOperators.GTE, map.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, map.maxLng).append(QueryOperators.GTE, map.minLng)); + def cursor = POI_COLL.find(q); + def resMap = [:]; + for (def dbo : cursor) { + def keyword = dbo.keyword; + def details = resMap.get(keyword); + if (!details) { + details = []; + resMap.put(keyword, details); + } + + def dMap = [:]; + dMap.name = dbo.name; + dMap.lat = dbo.lat; + dMap.lng = dbo.lng; + dMap.distance = com.fangjia.script.util.GeoUtils.calculateDistance(lat, lng, dbo.lat, dbo.lng); + details << dMap; + } + + resMap.each { k, v -> + println "[${k}] " + v; + } + return resMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 7555e8915..a31642613 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -566,6 +566,23 @@ class OpenApiUtils { return null; } + static def jsHouseCase(city, region, name, propertyTypeCode){ + def url = "http://localhost:8080/fangjia-open/house/case?city=${city}&district=${region}&buildingName=${name}&propertyTypeCode=${propertyTypeCode}" +// println url + String content = new HttpRequest().get(url, "utf-8"); + def dto = [] + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + }else{ + println url + } + } + return null; + } + static def financialRating(city, address){ def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" String content = new HttpRequest().get(url, "utf-8"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy index d2d11ec50..bd2f0760b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.commStat +import com.fangjia.core.RegionData import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils @@ -27,7 +28,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43") - +DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) //def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"] //for(def city : citys){ @@ -308,21 +309,65 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ //ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果") //区域级统计 -def city = "合肥" -def regions = CommonDatautils.getCityRegions(city) -def rows = [] -for(def region : regions){ - def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"]) - def alias_count = 0 - def addr_alias_count = 0 - for(def r : districts){ - if(r.alias && r.alias.size()>0){ - alias_count ++ +//def city = "合肥" +//def regions = CommonDatautils.getCityRegions(city) +//def rows = [] +//for(def region : regions){ +// def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"]) +// def alias_count = 0 +// def addr_alias_count = 0 +// for(def r : districts){ +// if(r.alias && r.alias.size()>0){ +// alias_count ++ +// } +// if(r.addr_alias && r.addr_alias.size()>0){ +// addr_alias_count++ +// } +// } +// rows << [region, districts.size(), alias_count, addr_alias_count] +//} +//ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果") + + +ExcelUtils.multiThreadingModify("D:\\数据\\临时\\区县楼盘量统计.xlsx",{d-> + if(d['C'] && !d['I']){ + def city = RegionData.formatCity(d['C']) + if(!city){ + city = RegionData.formatCity(d['B']) } - if(r.addr_alias && r.addr_alias.size()>0){ - addr_alias_count++ + if(city && d['D']){ + def region = RegionData.formatRegion(city, d['D']) + if(region){ + def count = CommonDatautils.findRegionDistrictCount("district", city, region) + println city +" "+ region+" "+count + d['I'] = city + d['J'] = region + d['K'] = count + } + // def dists = CommonDatautils.findAllDistricts(d['C'], "district", ["city","region","name"]) + //d['B'] = dists.size() } } - rows << [region, districts.size(), alias_count, addr_alias_count] -} -ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果") \ No newline at end of file + +// def q = new BasicDBObject() +// q.put("city",d['B']) +// q.put("district","") +// def dbo = mca_administrative_division_coll.findOne(q) +// if(!dbo){ +// q.put("city", "省直辖县级行政区划") +// q.put("district", d['B']) +// dbo = mca_administrative_division_coll.findOne(q) +// } +// if(dbo){ +// d['A']=dbo.province +// } +}) + +//def citys = ["北京","上海","重庆","天津","石家庄","唐山","秦皇岛","邯郸","邢台","保定","张家口","承德","沧州","廊坊","衡水","太原","大同","阳泉","长治","晋城","朔州","晋中","运城","忻州","临汾","吕梁","呼和浩特","包头","乌海","赤峰","通辽","鄂尔多斯","呼伦贝尔","巴彦淖尔","乌兰察布","兴安盟","锡林郭勒盟","阿拉善盟","沈阳","大连","鞍山","抚顺","本溪","丹东","锦州","营口","阜新","辽阳","盘锦","铁岭","朝阳","葫芦岛","长春","吉林","四平","辽源","通化","白山","松原","白城","延边","哈尔滨","齐齐哈尔","鸡西","鹤岗","双鸭山","大庆","伊春","佳木斯","七台河","牡丹江","黑河","绥化","大兴安岭","南京","无锡","徐州","常州","苏州","南通","连云港","淮安","盐城","扬州","镇江","泰州","宿迁","杭州","宁波","温州","嘉兴","湖州","绍兴","金华","衢州","舟山","台州","丽水","合肥","芜湖","蚌埠","淮南","马鞍山","淮北","铜陵","安庆","黄山","滁州","阜阳","宿州","六安","亳州","池州","宣城","福州","厦门","莆田","三明","泉州","漳州","南平","龙岩","宁德","南昌","景德镇","萍乡","九江","新余","鹰潭","赣州","吉安","宜春","抚州","上饶","济南","青岛","淄博","枣庄","东营","烟台","潍坊","济宁","泰安","威海","日照","临沂","德州","聊城","滨州","菏泽","郑州","开封","洛阳","平顶山","安阳","鹤壁","新乡","焦作","濮阳","许昌","漯河","三门峡","南阳","商丘","信阳","周口","驻马店","武汉","黄石","十堰","宜昌","襄阳","鄂州","荆门","孝感","荆州","黄冈","咸宁","随州","恩施","长沙","株洲","湘潭","衡阳","邵阳","岳阳","常德","张家界","益阳","郴州","永州","怀化","娄底","湘西","广州","韶关","深圳","珠海","汕头","佛山","江门","湛江","茂名","肇庆","惠州","梅州","汕尾","河源","阳江","清远","东莞","中山","潮州","揭阳","云浮","南宁","柳州","桂林","梧州","北海","防城港","钦州","贵港","玉林","百色","贺州","河池","来宾","崇左","海口","三亚","三沙","儋州","成都","自贡","攀枝花","泸州","德阳","绵阳","广元","遂宁","内江","乐山","南充","眉山","宜宾","广安","达州","雅安","巴中","资阳","阿坝州","甘孜州","凉山州","贵阳","六盘水","遵义","安顺","毕节","铜仁","黔西南","黔东南","黔南","昆明","曲靖","玉溪","保山","昭通","丽江","普洱","临沧","楚雄","红河","文山州","西双版纳州","大理","德宏州","怒江州","迪庆州","拉萨","日喀则","昌都","林芝","山南","那曲","阿里","西安","铜川","宝鸡","咸阳","渭南","延安","汉中","榆林","安康","商洛","兰州","嘉峪关","金昌","白银","天水","武威","张掖","平凉","酒泉","庆阳","定西","陇南","临夏","甘南","西宁","海东","海北州","黄南州","海南州","果洛州","玉树州","海西州","银川","石嘴山","吴忠","固原","中卫","乌鲁木齐","克拉玛依","吐鲁番","哈密","昌吉","博尔塔拉","巴音郭楞","阿克苏","克孜勒苏","喀什","和田","伊犁","塔城","阿勒泰","济源","仙桃","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","石河子","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉","昆玉"] +// +//def fjwCitys = CommonDatautils.findAllCity() +//for(def fjw_city : fjwCitys){ +// if(!citys.contains(fjw_city)){ +// println fjw_city +// } +//} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy index f05ae0163..f1336faa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -22,12 +23,16 @@ class ApiTest { static main(args) { def task = new ApiTest() -// task.rankRunTest() -// task.similarRunTest() - task.loupanLevelTest() +// task.rankRunTest()//小区排名接口测试 +// task.similarRunTest()//相似房源接口测试 +// task.loupanLevelTest()//楼盘评级接口测试 +// task.jsAbcHouseCastTest()//江苏农总行房源接口测试 + task.arroundFacilities()//周边poi } + + def loupanLevelTest(){ ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d -> if (d['A']) { @@ -125,6 +130,150 @@ class ApiTest { return dbo } + def jsAbcHouseCastTest(){ + def citys = ["无锡"] + for(def city : citys){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","estate_type2"]) + def rets = getHouseCases(districts) + ExcelUtils.write("D:\\数据\\江苏\\房源样本.xlsx", rets, city) + } + + } + + def getHouseCases(districts){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + rets << ["城市","区域","名称", "物业类型", "最小案例","最大案例", "案例"] + for(def i=0;i + if(!row['L']){ + def city = row['C'] + def region = row['D'] + def name = row['F'] + def address = row['G'] + + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, "property", false) + if(jsonObject && !jsonObject.containsKey("result")){ + jsonObject = FangjiaClientUtils.newMatchPrice(city, address, "property", false) + } + if(jsonObject && !jsonObject.containsKey("result")){ + jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, null, true) + } + + if(jsonObject && !jsonObject.containsKey("result")){ + jsonObject = FangjiaClientUtils.newMatchPrice(city, address, null, true) + } + if(jsonObject && jsonObject.containsKey("result")) { + def resultMap = jsonObject.get("result") + row['L'] = resultMap.get("city") + row['M'] = resultMap.get("district") + row['N'] = resultMap.get("name") + def type = resultMap.get("propertyType") + if(type && type!='null'){ + row['O'] = type + } + if(resultMap.get("avgPrice")){ + row['P'] = resultMap.get("avgPrice") + } + } + } + }) + } + def addPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy new file mode 100644 index 000000000..544d54541 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -0,0 +1,286 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.RegionData +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.* +import java.util.concurrent.atomic.AtomicLong + +class SuggestionDataSynch extends BaseBatch{ + + static def targetUrl = "http://192.168.0.12:9200/seaweed/" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) + DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) + + public static def provinceMap = CommonDatautils.findAllCityProvince() + + def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + public static def priceMap = null + + static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + + + public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) + + def keys = new CopyOnWriteArraySet(); + def fullCityMap = null + def mcaDistMap = null + + static main(args) { + runBatch(new SuggestionDataSynch(), args); + } + + @Override + public void process(Object args) { + run() + } + + + + def run() { + //"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆" + def citys = CommonDatautils.findAllCity() + def index = 0 + fullCityMap = queryFullNameCity() + mcaDistMap = queryDistNameCity() + + for (def c : citys) { + index++ + if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ + updating(c, "district") + updating(c, "office") + updating(c, "shop") + println index + " " + c + " 完成" + + } + +// poiData(c) + + } + } + + def updating(city, cat){ + def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"]) + def rets = new CopyOnWriteArrayList() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i0){ + map.alias = dist.alias + } + if(dist.addr_alias && dist.addr_alias.size()>0){ + map.addr_alias = dist.addr_alias + } + map.type = dist.cat + if(dist.lng){ + def loc = [:] + loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0) + loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0) + map.location = loc + } + map.source = "FJW" + return map + } + + + + def findAllDistrictSellPrice(){ + def q = new BasicDBObject() + q.put("date", lastMonthDate) + def priceMap = [:] + DISTRICT_SELL_STAT.find(q).each{ + priceMap.put(it.commid, it) + } + return priceMap + } + + + def updateData(map){ + def udata = new BasicDBObject() + map.each{k,v-> + if(k!="id"){ + udata.put(k,v) + } + } + SUGGESTION_CONN.update(new BasicDBObject('_id', map.id), new BasicDBObject('$set', udata), true, false) + } + + + def poiData(city){ + def data = fullCityMap.get(city) + if(data){ + def results = CommonDatautils.findAmapDetailBySql(data.city) + createPoi(results, data) + println city +" 完成!" + } + + } + + + def createPoi(results, data){ + def rets = new CopyOnWriteArrayList() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + for(def i=0;i CITY_PROVINCE_MAP = new HashMap(); static DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com", 27017); + + static final Map PROVINCE_MAP = new HashMap() {{ + put("河北", "河北省"); + put("河南", "河南省"); + put("云南", "云南省"); + put("辽宁", "辽宁省"); + put("黑龙江", "黑龙江省"); + put("湖南", "湖南省"); + put("湖北", "湖北省"); + put("安徽", "安徽省"); + put("山东", "山东省"); + put("新疆", "新疆维吾尔自治区"); + put("江苏", "江苏省"); + put("浙江", "浙江省"); + put("江西", "江西省"); + put("广西", "广西壮族自治区"); + put("甘肃", "甘肃省"); + put("山西", "山西省"); + put("内蒙古", "内蒙古自治区"); + put("陕西", "陕西省"); + put("吉林", "吉林省"); + put("福建", "福建省"); + put("贵州", "贵州省"); + put("广东", "广东省"); + put("青海", "青海省"); + put("西藏", "西藏自治区"); + put("四川", "四川省"); + put("宁夏", "宁夏回族自治区"); + put("海南", "海南省"); + put("台湾", "台湾省"); + }}; + static { logger.info("开始加载省份信息"); DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("province", 1).append("display", 1)).batchSize(300); @@ -109,6 +142,16 @@ public class ProvinceData { return CITY_PROVINCE_MAP.get(city); } + + /** + * 根据省的简称获取省的全称 + * @param province + * @return + */ + public static String getFullProvince(String province){ + return PROVINCE_MAP.get(province); + } + /** * 获取省份下的所有城市 * @param province diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 7b199ce7c..6f44f405d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,6 +11,7 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def fileStartDate = DateUtil.parseDate("20230201", "yyyyMMdd") def startDate = DateUtil.parseDate("20230201", "yyyyMMdd") def endDate = DateUtil.parseDate("20230301", "yyyyMMdd") Calendar calendar = Calendar.getInstance() @@ -38,7 +39,7 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志") +ExcelUtils.write("D:\\亲邻数据${fileStartDate.format('yyyyMM')}.xlsx", data, "查询日志") //def list = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy index 778cce630..582af4f11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230901", "yyyyMMdd") -def endDate = DateUtil.parseDate("20231001", "yyyyMMdd") +def startDate = DateUtil.parseDate("20231001", "yyyyMMdd") +def endDate = DateUtil.parseDate("20231101", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -39,6 +39,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202310.xlsx", data, "楼盘评情") +ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202311.xlsx", data, "楼盘评情")