From 3821473e2823ffc8144a5e0713d2e42ed5a7997a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 19 Aug 2022 15:19:19 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 35 ++++ .../script/ccbevaluation/TradeStatLog.groovy | 12 +- .../script/commData/CommonDatautils.groovy | 20 +- .../script/fuzhou/FzDataResultStat.groovy | 11 +- .../script/fuzhou/FzDistrictMatch.groovy | 176 ++++++++++++++++++ .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fuzhou/ThirdPriceExportBatch.groovy | 65 ++++--- .../fangjia/script/gj/YouDaDataParser.groovy | 6 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 6 +- .../script/gj/paimai/ExportPaimai.groovy | 11 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 6 +- .../script/gj/test/DataStatTest.groovy | 21 +-- .../script/gj/test/MingShengStat.groovy | 160 ++++++++++++++++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 5 +- 16 files changed, 478 insertions(+), 68 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 12a02fabb..407d37700 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 08ab70333..059531488 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -42,6 +42,10 @@ class ExportOracleData { exportBigdataDeal("2022-06-01", new_dists) + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) @@ -93,6 +97,37 @@ class ExportOracleData { } + def exportWebsiteRate(){ + def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.DATA_SOURCE_MAP + data << r.WEIGHT + data << r.TYPE + data << r.SOURCECODE + data << r.CITYID + data << r.CITYNAME + data << r.CREATETIME + data << r.UPDATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + def exportBigdataPoiData() { def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj"; def allRows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index ee37a0616..4a38d4ca3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-04-01" - def date = "2022-05-01" - def endDate = "2022-06-01" - def newDate = "2022-07-01" + def startDate = "2022-05-01" + def date = "2022-06-01" + def endDate = "2022-07-01" + def newDate = "2022-08-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\${newDate}成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index fede45dfc..94c151d1a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -70,7 +70,7 @@ class CommonDatautils { public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl) public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl) - + public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -136,6 +136,13 @@ class CommonDatautils { return cityInfos; } + + + public static def findAlCityDetail(city){ + DBObject q = new BasicDBObject(); + q.put("display", city) + return SERVER_AREA_COLL.findOne(q); + } public static def findAllCityByGrade(def city_grade){ DBObject q = new BasicDBObject(); @@ -3313,5 +3320,16 @@ class CommonDatautils { def count = HOUSE_LOUDONG_COLL.count(q) return count; } + + static def findCityInfo(city){ + def q = new BasicDBObject() + q.put("city", city) + def cur = CITY_INFO_COLL.find(q).sort(new BasicDBObject("weekend",-1)).limit(1) + while(cur.hasNext()) { + DBObject bsobj = cur.next() + return bsobj; + } + return null + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 778312b6b..cad3879ac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -24,7 +24,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() - task.priceMarkupRun(); +// task.priceMarkupRun(); task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() @@ -88,9 +88,14 @@ class FzDataResultStat { lastPrice = lastObj[5] } def markup = null - if(list && lastPrice){ - markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + try{ + if(list && lastPrice && lastPrice.toInteger()>0){ + markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + } + }catch(Exception e){ + e.printStackTrace() } + def dist = CommonDatautils.findDistrictById(list[1]) def flag = "有效" if(dist.status!=0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy new file mode 100644 index 000000000..cae471308 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -0,0 +1,176 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.io.FileUtils + +class FzDistrictMatch { + + + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + + + static main(args) { + def task = new FzDistrictMatch() +// task.run(); +// task.addPrice() + task.addThirdPrice() + } + + def addThirdPrice(){ + def map = read("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv"); + ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d -> + if(d['G']){ + def priceMap = map.get(d['G']) + def prices = [] + if(priceMap && priceMap.zhugezhaofang && priceMap.zhugezhaofang.isNumber()){ + d['J'] = priceMap.zhugezhaofang + prices << priceMap.zhugezhaofang.toInteger() + } + + if(priceMap && priceMap.anjuke && priceMap.anjuke.isNumber()){ + d['K'] = priceMap.anjuke + prices << priceMap.anjuke.toInteger() + } + + if(priceMap && priceMap.soufang && priceMap.soufang.isNumber()) { + d['L'] = priceMap.soufang + prices << priceMap.soufang.toInteger() + } + + if(priceMap && priceMap.lianjia && priceMap.lianjia.isNumber()){ + d['M'] = priceMap.lianjia + prices << priceMap.lianjia.toInteger() + } + + if(priceMap && priceMap.cityhouse && priceMap.cityhouse.isNumber()){ + d['N'] = priceMap.cityhouse + prices << priceMap.cityhouse.toInteger() + } + + if(priceMap && priceMap.fangtan && priceMap.fangtan.isNumber()){ + d['O'] = priceMap.fangtan + prices << priceMap.fangtan.toInteger() + } + + if(priceMap && priceMap.fangdoudou && priceMap.fangdoudou.isNumber()){ + d['P'] = priceMap.fangdoudou + prices << priceMap.fangdoudou.toInteger() + } + if(priceMap && priceMap.maitian && priceMap.maitian.isNumber()){ + d['Q'] = priceMap.maitian + prices << priceMap.maitian.toInteger() + } + + if(priceMap && priceMap.get("58tc") && priceMap.get("58tc").isNumber()){ + d['R'] = priceMap.get("58tc") + prices << priceMap.get("58tc").toInteger() + } + if(prices){ + d['S'] = (int)(prices.sum()/prices.size()) + } + } + },0) + } + + def read(fileName){ + List list1= FileUtils.readLines(new File(fileName)) + def map1 = [:] + for(def str : list1){ + def s = str.split("@\\|@") + def key = s[2]+"##"+s[3]+"##"+s[4] + def priceMap = [:] + map1.put(s[1], priceMap) + + if(s.size()>=7 && s[6]){ + priceMap.put("zhugezhaofang", s[6]) + } + + if(s.size()>=8 && s[7]){ + priceMap.put("anjuke", s[7]) + } + + if(s.size()>=9 && s[8]){ + priceMap.put("soufang", s[8]) + } + + if(s.size()>=10 && s[9]){ + priceMap.put("lianjia", s[9]) + } + + if(s.size()>=11 && s[10]){ + priceMap.put("cityhouse", s[10]) + } + + if(s.size()>=12 && s[11]){ + priceMap.put("fangtan", s[11]) + } + + if(s.size()>=13 && s[12]){ + priceMap.put("fangdoudou", s[12]) + } + if(s.size()>=14 && s[13]){ + priceMap.put("maitian", s[13]) + } + + try{ + if(s.size()>=15 && s[14]){ + priceMap.put("58tc", s[14]) + } + }catch(Exception e){ + e.getMessage() + } + + } + return map1 + } + + def addPrice(){ + ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d -> + if(d['G']){ + def avg_price = findFzDistrictStat( d['G'],202207) + d['F'] = avg_price + } + },0); + } + + + def findFzDistrictStat(id,lastMonthDate){ + def q = new BasicDBObject() + q.put("comm_id", id) + q.put("month", lastMonthDate) + def dbcursor = FZ_DISTRICT_STAT_COLL.find(q) + for(def d : dbcursor){ + return d.avg_price + } + return null + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx",{d-> + def region = d['C'] + if(!d['G'] && region){ + def name = d['B'].toString().replace("(","").replace(")","") +// name = d['B'].toString().replace("(非钢混)","").replace("(钢混)","") + region = RegionData.formatRegion("福州", region) +// def dist = CommonDatautils.findDistrict("福州", region, name, "district") +// if(dist){ +// d['G'] = dist._id.toString() +// d['H'] = dist.name +// } + def jsonObject = OpenApiClient.addressMatch("福州", name, "property", false, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + d['H'] = jsonObject.get("name") + d['G'] = jsonObject.get("id") + d['J'] = jsonObject.get("district") + } + + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 43e47801d..c9a2c19be 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index 9e3a6dd7e..ac8449aa8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -75,81 +75,98 @@ class ThirdPriceExportBatch extends BaseBatch{ row << dist.region row << dist.name row << o.s_date + def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice) row << zgf_price if(zgf_price){ add = true; } -// if(o.zhugezhaofang_esf_price){ -// add = true; -// } + def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice) row << ajk_price if(ajk_price){ add = true; } -// if(o.anjuke_esf_price){ -// add = true; -// } def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice) row << sf_price if(sf_price){ add = true; } -// if(o.soufang_esf_price){ -// add = true; -// } + def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice) row << lj_price if(lj_price){ add = true; } -// if(o.lianjia_esf_price){ -// add = true; -// } def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice) row << city_price if(city_price){ add = true; } -// if(o.cityhouse_esf_price){ -// add = true; -// } def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice) row << ft_price if(ft_price){ add = true; } -// if(o.fangtan_esf_price){ -// add = true; -// } def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice) row < +ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d-> // def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) // if(keys){ -// d['E'] = keys.size() +// d['G'] = keys.size() // } // def rets = findHousesByCity(d['A'], "sell", 20210701) if(d['A']){ def count = findHouseCount(d['A']) - d['F'] = count + d['H'] = count } }) @@ -288,7 +288,7 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ //} // -//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"] + //def rows = [] //rows << ["城市","小区量", "别名量", "地址别名量"] //for(def c : citys){ @@ -305,7 +305,4 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ // } // rows << [c, districts.size(), alias_count, addr_alias_count] //} -// -// -// -//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果") +//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy new file mode 100644 index 000000000..089d4c535 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MingShengStat.groovy @@ -0,0 +1,160 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class MingShengStat { + + def cityLevel0 = ["北京","上海","广州","深圳"] + def cityLevel1 = ["成都","重庆","杭州","武汉","西安","郑州","青岛","长沙","天津","苏州","南京","东莞","沈阳","合肥","佛山"] + def cityLevel2 = ["昆明","福州","无锡","厦门","哈尔滨","长春","南昌","济南","宁波","大连","贵阳","温州","石家庄","泉州","南宁","金华","常州","珠海","惠州","嘉兴","南通","中山","保定","兰州","台州","徐州","太原","绍兴","烟台","廊坊"] + def cityLevel3 = ["海口","汕头","潍坊","扬州","洛阳","乌鲁木齐","临沂","唐山","镇江","盐城","湖州","赣州","漳州","揭阳","江门","桂林","邯郸","泰州","济宁","呼和浩特","咸阳","芜湖","三亚","阜阳","淮安遵义","银川","衡阳","上饶","柳州","淄博","莆田","绵阳","湛江","商丘","宜昌", + "沧州","连云港","南阳","蚌埠","驻马店","滁州","邢台","潮州","秦皇岛","肇庆","荆州","周口","马鞍山","清远","宿州","威海","九江","新乡","信阳","襄阳","岳阳","安庆","菏泽","宜春","黄冈","泰安","宿迁","株洲","宁德","鞍山","南充","六安","大庆","舟山"] + def cityLevel4 = ["常德","渭南湖","孝感","丽水","运城","德州","张家口","鄂尔多斯","阳江","泸州","丹东","曲靖","乐山","许昌","湘潭","晋中","安阳","齐齐哈尔","北海","宝鸡","抚州","景德镇","延安","三明","抚顺亳州","日照","西宁","衢州","拉萨","淮北","焦作","平顶山","滨州","吉安","濮阳","眉山","池州","荆门","铜仁","长治", + "衡水","铜陵","承德","达州","邵阳","德阳","龙岩","南平","淮南","黄石","营口","东营","吉林","韶关","枣庄","包头","怀化","宣城","临汾","聊城","梅州","盘锦","锦州","榆林","玉林","十堰","汕尾","咸宁","宜宾","永州","益阳","黔南州","黔东南","恩施","红河","大理","大同","鄂州","忻州","吕梁","黄山","开封","郴州","茂名","漯河","葫芦岛","河源","娄底","延边"] + def cityLevel5 = ["汉中","辽阳","四平","内江","六盘水","安顺","新余","牡丹江","晋城","自贡","三门峡","赤峰","本溪","防城港","铁岭","随州","广安","广元","天水","遂宁","萍乡","西双版纳","绥化","鹤壁","湘西","松原","阜新","酒泉","张家界","黔西南","保山","昭通","河池","来宾","玉溪","梧州","鹰潭","钦州","云浮","佳木斯","克拉玛依","呼伦贝尔","贺州", + "通化","阳泉","朝阳","百色","毕节","贵港","丽江","安康","通辽","德宏","朔州","伊犁","文山","楚雄","嘉峪关","凉山","资阳","锡林郭勒盟","雅安","普洱","崇左","庆阳","巴音郭楞(巴州)","乌兰察布","白山","昌吉","白城","兴安盟","定西","喀什","白银","陇南","巴彦淖尔","巴中","鸡西","乌海","临沧","海东","张掖","商洛","黑河","哈密","吴忠","阿克苏"] + + def ajkNewDistsMap = ["北京": 538, "上海": 1928, "广州": 1834, "深圳": 1149, "武汉": 2181, "大连": 866, "南京": 1442, "杭州": 2245, "太原": 497, "石家庄": 723, "重庆": 2700, "西安": 1693, "福州": 305, "济南": 1003, "宁波": 1304, "成都": 3004, "天津": 1614, "昆明": 755, "南昌": 697, "青岛": 1843, "厦门": 253, "郑州": 1385, "长沙": 1811, "长春": 741, "合肥": 1265, "呼和浩特": 462, "沈阳": 456, "贵阳": 499, "拉萨": 82, "宝鸡": 249, "汕头": 411, "苏州": 1531, "温州": 566, "泉州": 565, "邯郸": 314, "唐山": 442, "沧州": 248, "洛阳": 560, "南阳": 422, "襄阳": 302, "吕梁": 79, "大同": 171, "潍坊": 739, "南通": 735, "衡阳": 425, "中山": 590, "宜昌": 360, "许昌": 384, "盐城": 459, "济宁": 510, "龙岩": 227, "上饶": 288, "马鞍山": 199, "赣州": 385, "金华": 479, "常德": 394, "延边": 71, "泰州": 304, "三亚": 456, "淮安": 488, "咸宁": 307, "无锡": 703, "常州": 470, "镇江": 382, "烟台": 917, "绍兴": 714, "莆田": 261, "东莞": 729, "佛山": 1161, "株洲": 285, "黄石": 151, "孝感": 379] + def bkNewDistsMap = ["北京": 316, "上海": 2002, "广州": 1350, "深圳": 1261, "武汉": 2308, "大连": 438, "南京": 579, "杭州": 1892, "太原": 418, "石家庄": 582, "重庆": 2122, "西安": 1608, "福州": 512, "济南": 779, "宁波": 592, "成都": 2849, "天津": 1039, "昆明": 1262, "南昌": 429, "青岛": 1275, "厦门": 387, "郑州": 1146, "长沙": 1434, "长春": 433, "合肥": 807, "呼和浩特": 294, "沈阳": 1137, "贵阳": 242, "拉萨": 3, "宝鸡": 50, "苏州": 1129, "温州": 126, "泉州": 410, "邯郸": 114, "唐山": 181, "沧州": 50, "洛阳": 192, "南阳": 290, "襄阳": 55, "大同": 62, "潍坊": 332, "南通": 509, "衡阳": 83, "中山": 516, "宜昌": 147, "许昌": 156, "盐城": 211, "济宁": 205, "龙岩": 5, "上饶": 160, "马鞍山": 120, "赣州": 236, "金华": 124, "常德": 116, "三亚": 164, "淮安": 278, "咸宁": 75, "无锡": 575, "常州": 221, "镇江": 310, "烟台": 548, "绍兴": 175, "莆田": 13, "东莞": 622, "佛山": 971, "株洲": 134, "黄石": 88, "孝感": 94] + def ljNewDistMap = ["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40] + def sfNewDistMap =["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40] + def chNewDistMap = ["北京": 408, "上海": 532, "广州": 559, "深圳": 763, "武汉": 775, "大连": 7, "南京": 1137, "杭州": 517, "太原": 293, "石家庄": 567, "重庆": 817, "西安": 439, "济南": 544, "宁波": 555, "成都": 1094, "天津": 754, "昆明": 446, "南昌": 379, "青岛": 840, "厦门": 267, "郑州": 960, "长沙": 767, "长春": 299, "合肥": 404, "呼和浩特": 136, "沈阳": 385, "贵阳": 330, "拉萨": 41, "宝鸡": 145, "汕头": 220, "苏州": 715, "温州": 422, "泉州": 312, "邯郸": 425, "唐山": 317, "沧州": 298, "洛阳": 268, "南阳": 233, "襄阳": 84, "吕梁": 38, "大同": 121, "潍坊": 421, "南通": 328, "衡阳": 329, "中山": 279, "宜昌": 158, "许昌": 161, "盐城": 165, "济宁": 251, "龙岩": 90, "上饶": 136, "马鞍山": 69, "赣州": 265, "金华": 252, "常德": 192, "延边": 59, "泰州": 122, "三亚": 120, "淮安": 150, "咸宁": 175, "无锡": 468, "常州": 259, "镇江": 223, "烟台": 420, "绍兴": 368, "莆田": 106, "东莞": 249, "佛山": 438, "株洲": 216, "黄石": 96, "孝感": 160] + def jwNewDistMap = ["北京": 510, "上海": 1353, "广州": 1193, "深圳": 697, "武汉": 1746, "大连": 539, "南京": 1219, "杭州": 2030, "太原": 631, "石家庄": 521, "重庆": 2165, "西安": 1518, "济南": 811, "宁波": 1023, "成都": 3192, "天津": 91, "昆明": 764, "南昌": 632, "青岛": 1420, "厦门": 274, "郑州": 1509, "长沙": 1474, "长春": 400, "合肥": 1112, "呼和浩特": 320, "沈阳": 648, "贵阳": 501, "拉萨": 75, "宝鸡": 217, "汕头": 350, "苏州": 1210, "温州": 628, "泉州": 407, "邯郸": 203, "唐山": 638, "沧州": 367, "洛阳": 509, "南阳": 311, "襄阳": 206, "吕梁": 48, "大同": 159, "潍坊": 1041, "南通": 883, "衡阳": 420, "中山": 457, "宜昌": 319, "许昌": 151, "盐城": 488, "济宁": 534, "龙岩": 161, "上饶": 205, "马鞍山": 181, "赣州": 380, "金华": 424, "常德": 305, "延边": 102, "泰州": 172, "三亚": 460, "淮安": 368, "咸宁": 223, "无锡": 509, "常州": 420, "镇江": 506, "烟台": 976, "绍兴": 608, "莆田": 226, "东莞": 1270, "佛山": 894, "株洲": 274, "黄石": 170, "孝感": 233] + def zgNewDistMap = ["南京": 711, "大连": 143, "青岛": 1117, "济南": 577, "昆明": 726, "中山": 466, "佛山": 852, "盐城": 384, "北京": 563, "无锡": 342, "东莞": 428, "烟台": 853, "厦门": 289, "南通": 325, "太原": 310, "泉州": 354, "贵阳": 1324, "马鞍山": 141, "上海": 1776, "苏州": 385, "郑州": 610, "常州": 230, "洛阳": 228, "潍坊": 795, "赣州": 238, "淮安": 268, "深圳": 1034, "长沙": 1252, "石家庄": 296, "金华": 156, "南昌": 383, "泰州": 255, "广州": 1193, "三亚": 341, "宁波": 475, "唐山": 382, "镇江": 495, "济宁": 330, "莆田": 213, "沧州": 155, "宝鸡": 173, "重庆": 634, "沈阳": 494, "长春": 182, "襄阳": 164, "天津": 556, "合肥": 366, "宜昌": 252, "温州": 428, "南阳": 379, "成都": 2452, "绍兴": 147, "呼和浩特": 337, "武汉": 845, "西安": 1407, "邯郸": 220, "常德": 259, "杭州": 568, "黄石": 143, "大同": 151, "许昌": 289, "延边": 72, "咸宁": 230, "拉萨": 60, "株洲": 225, "上饶": 210, "孝感": 229, "吕梁": 78, "汕头": 329, "龙岩": 150, "衡阳": 328] + + def fields = ["city","region","district_name","estate_type","area","avg_price","total_price","direction","fitment","room","hall","toilet","floor","height","build_structure","comm_id","dataType1","purpose","root","url", "p_date"] + def no_fields = ["build_date","build_type","floor_info"] + static main(args) { + def task = new MingShengStat(); +// task.run() +// task.addNewLevel + task.addHouseRate() +// task.houseFieldStat() + } + + def houseFieldStat(){ + def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"] + def rows = [] + for(def c : citys){ + def row = [] + row << c + def total = findHouseCount(c) + for(def f : no_fields){ + def num = findValueByCity(c, f); + if(num){ + def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total) + row << rate + }else{ + row << 0 + } + } + rows << row + } + + ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计2.xlsx", rows, "字段统计") + } + + def addHouseRate(){ + // def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\估值数据统计结果20220816.xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\民生\\城市统计.xlsx",{d-> + if(d['B']){ + def cityInfoDto = CommonDatautils.findCityInfo(d['B']) + d['G'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, cityInfoDto.price_coverage*100) + def dists = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city","region","name"]) + d['F'] = dists.size() + d['D'] = statNewDistCount(d['B']) + d['E'] = findHouseCount(d['B']) + } + }) + } + + def findValueByCity(city, field){ + def count = 0 + def sql = "select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell' and ${field} is not null" + ElasticUtils.executeSqlQuery(sql, { + count = it.count + println("$city\t${field} ${it.count}") + }) + return count + } + + def findHouseCount(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell'", { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def statNewDistCount(def city){ + def sum = 0 + if(ajkNewDistsMap.get(city)){ + sum = sum + ajkNewDistsMap.get(city) + } + if(bkNewDistsMap.get(city)){ + sum = sum + bkNewDistsMap.get(city) + } + if(ljNewDistMap.get(city)){ + sum = sum + ljNewDistMap.get(city) + } + if(sfNewDistMap.get(city)){ + sum = sum + sfNewDistMap.get(city) + } + if(chNewDistMap.get(city)){ + sum = sum + chNewDistMap.get(city) + } + if(jwNewDistMap.get(city)){ + sum = sum + jwNewDistMap.get(city) + } + if(zgNewDistMap.get(city)){ + sum = sum + zgNewDistMap.get(city) + } + return sum + } + + def addNewLevel(){ + ExcelUtils.modify("D:\\数据\\民生\\城市统计.xlsx",{d-> + if(d['B']){ + if(cityLevel0.contains(d['B'])){ + d['D'] = "一线" + }else if(cityLevel1.contains(d['B'])){ + d['D'] = "新一线" + }else if(cityLevel2.contains(d['B'])){ + d['D'] = "二线" + }else if(cityLevel3.contains(d['B'])){ + d['D'] = "三线" + }else if(cityLevel4.contains(d['B'])){ + d['D'] = "四线" + }else if(cityLevel5.contains(d['B'])){ + d['D'] = "五线" + } + } + }) + } + + def run(){ + def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"] + def rows = [] + for(def c : citys){ + def dbo = CommonDatautils.findAlCityDetail(c) + def row = [] + if(dbo){ + row << dbo.province + row << c + row << dbo.new_city_grade/1000 + }else{ + row << "" + row << c + row << "" + } + rows << row + } + ExcelUtils.write("D:\\数据\\民生\\城市统计.xlsx", rows, "数据") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 6ff3f087f..0b7d904a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "城市房产"// - def citys = findCrawlCity(source, 202205); + def source = "365淘房"// + def citys = findCrawlCity(source, 202206); for(def city : citys){ - def count1 = findCount(source, city, 202205); - def count2 = findCount(source, city, 202206); + def count1 = findCount(source, city, 202206); + def count2 = findCount(source, city, 202207); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 91e44595b..1f2cbdb96 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -23,7 +23,8 @@ class SynchDistrictPrice { def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] //,"城市房产":"城市房产" - def esSources = ["365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","吉屋":"吉屋"]// + //"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房", + def esSources = ["我爱我家":"我爱我家","安居客小程序":"安居客","搜房":"搜房","贝壳找房":"贝壳找房",]//"吉屋":"吉屋","365淘房":"365淘房" static main(args) { def task = new SynchDistrictPrice() // task.run() @@ -46,7 +47,7 @@ class SynchDistrictPrice { def findAllEsData(city,esSource, mgSource){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s