From 2828feab7cb95ecaa0b85f9375346416115a59de Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Jul 2022 16:40:58 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 66 +++++++++--------- .../ccbevaluation/BigdataForsaleBatch.groovy | 7 +- .../script/ccbevaluation/TradeStatLog.groovy | 12 ++-- .../script/commData/CommonDatautils.groovy | 39 ++++++++++- .../fuzhou/DistrictPriceExportBatch.groovy | 3 +- .../script/fuzhou/FzDataResultStat.groovy | 35 +++++++--- .../script/fuzhou/HouseExportBatch.groovy | 11 +-- .../com/fangjia/script/fuzhou/Test1.groovy | 6 +- .../fangjia/script/gj/YouDaDataParser.groovy | 6 +- .../script/gj/crawl/AnjukeAppCrawl.groovy | 4 +- .../script/gj/crawl/AnjukeHouseCrawl.groovy | 4 +- .../script/gj/crawl/AnjukeSendMq.groovy | 10 ++- .../script/gj/data/RegionPriceExport.groovy | 32 ++++++--- .../gj/district/FangjiaClientUtils.groovy | 4 +- .../script/gj/level/DistrictIndexBatch.groovy | 3 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 21 ++++-- .../script/gj/paimai/ExportPaimai.groovy | 69 ++++++++++++++++--- .../gj/paimai/LianjiaCrawlByChrome.groovy | 12 ++-- .../script/gj/paimai/PaimaiDataSend.groovy | 41 +++++------ .../script/gj/paimai/PaimaiTest.groovy | 34 ++++++++- .../com/fangjia/script/gj/test/Test.groovy | 22 ++++-- .../script/thirdPriceCrawl/DataStat.groovy | 8 +-- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 2 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 21 +++--- 24 files changed, 326 insertions(+), 146 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index c1ea41780..da60a19f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -60,39 +60,39 @@ class BigdataDealBatch extends BaseBatch{ addDealPriceMatch(city); // } } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - for(def city : citys){ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - - def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成" +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// for(def city : citys){ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// +// def trades = getCityData(city, statMap, ppeDists) +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 724682101..7088dea02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220518, 20220619); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) @@ -454,6 +454,11 @@ class BigdataForsaleBatch extends BaseBatch{ println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type continue; } + + if(seawater.estate_type&& seawater.estate_type.contains("别墅")){ + println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type + continue; + } } if(!seawater.total_price){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index fc73e4867..ee37a0616 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","3月总量","4月总量","5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-03-01" - def date = "2022-04-01" - def endDate = "2022-05-01" - def newDate = "2022-06-01" + def startDate = "2022-04-01" + def date = "2022-05-01" + def endDate = "2022-06-01" + def newDate = "2022-07-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-05成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 38d9b23b3..fede45dfc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -877,6 +877,8 @@ class CommonDatautils { keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) keys.put("sid", 1) + keys.put("status", 1) + try{ DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ @@ -2278,6 +2280,24 @@ class CommonDatautils { return prices; } + public static def queryDistrictPrices(city, region, startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))); + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + def prices = []; + while(cur.hasNext()) { + DBObject dbo = cur.next() + def price = dbo.get("avg_price"); + if(price){ + prices << price; + } + } + + return prices; + } + public static def queryDistrictPricesByDate(city, region, name, startDate, endDate){ DBObject queryCondition = new BasicDBObject() @@ -3031,9 +3051,26 @@ class CommonDatautils { return price; } + static def queryCityRegionPrices(city, region, startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("block", null); + queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))); + DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def prices = []; + while(cur.hasNext()) { + DBObject dbo = cur.next() + prices << dbo.get("avg_price"); + } + return prices; + } + + + - static def queryCityRegionBlockPrice(city, region, block, startDate, endDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index dd9cd246b..c38bb7346 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -129,7 +129,7 @@ class DistrictPriceExportBatch extends BaseBatch{ if(curPrice){ m << dbo.get("_id").toString()//均价ID }else{ - m << "" + m << new ObjectId().toString() } m << rMap.get("_id").toString()//楼盘ID m << rMap.city @@ -210,6 +210,7 @@ class DistrictPriceExportBatch extends BaseBatch{ }else{ def udata = new BasicDBObject() def o = new BasicDBObject() + println r[0] o.put("_id", new ObjectId(r[0])) o.put("comm_id", r[1]) o.put("city", r[2]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 81c5624d1..778312b6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -55,11 +56,17 @@ class FzDataResultStat { List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def distkeys = [] def map1 = [:] for(def s : list1){ def row = s.split("@\\|@") def key = row[2]+"##"+row[3]+"##"+row[4] - map1.put(key, row[5]) + if(!distkeys.contains(key)){ + distkeys << key + }else{ + println "重复:"+key + } + map1.put(key, row) } List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) @@ -67,19 +74,29 @@ class FzDataResultStat { for(def s : list2) { def row = s.split("@\\|@") def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row[5]) + map2.put(key, row) } def rows = [] - rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] - map1.each {k,v-> + rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] + map1.each {k,list-> def row = [] def keys = k.split("##") - def lastPrice = map2.get(k) - def markup = null - if(v && lastPrice){ - markup = (v.toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + + def lastObj = map2.get(k) + def lastPrice = null + if(lastObj){ + lastPrice = lastObj[5] } - rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup] + def markup = null + if(list && lastPrice){ + markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + } + def dist = CommonDatautils.findDistrictById(list[1]) + def flag = "有效" + if(dist.status!=0){ + flag = "失效" + } + rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 24a397d3e..b5aa0e611 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -73,8 +73,8 @@ class HouseExportBatch extends BaseBatch{ def fileName = "/fzbank/output/FJW_HOUSE_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_HOUSE_"+curDate+".sha512" if(CommonDatautils.isWindow()){ - fileName = "F:\\数据\\福州\\FJW_HOUSE_"+curDate+".csv" - dataCodeFileName = "F:\\数据\\福州\\FJW_HOUSE_"+curDate+".sha512" + fileName = "D:\\数据\\福州\\FJW_HOUSE_"+curDate+".csv" + dataCodeFileName = "D:\\数据\\福州\\FJW_HOUSE_"+curDate+".sha512" } batchCreateFile(ret, fileName, headers, charset); println "数据文件输出完成......" @@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{ // int endDate = 20191201 def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd") - def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); - + // def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); + def endDate = "20220710" QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); // QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false); @@ -420,7 +420,8 @@ class HouseExportBatch extends BaseBatch{ row << seawater.name row << seawater.url - row << seawater.p_date +// row << seawater.p_date + row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") rets << row } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 563e6ab47..43e47801d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202205.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -136,8 +136,8 @@ def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计") def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s -//def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -//DingDingMessageUtil.sendTextMessage(access_token, s) +def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" +DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 68494ea8b..4f8e69217 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-03-01" + static def statDate = "2022-06-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo - task.esfParser();//友达二手房存入mongo +// task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年7月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index 21ddf507c..b97ec6bb1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -44,7 +44,7 @@ class AnjukeAppCrawl { // String line2 = StringEscapeUtils.unescapeJava(s); // println line2 def task = new AnjukeAppCrawl(); -// task.autoRun() + task.autoRun() // task.run(); // task.run2() @@ -55,7 +55,7 @@ class AnjukeAppCrawl { // task.groupByCity() // task.selectNoSave() // task.statCount() - task.exportFzDist() +// task.exportFzDist() } def saveToEs1(){ def lists = getAllAppDatas() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index bd7567c8e..065010b5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,11 +25,11 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://fz.anjuke.com/prop/view/S2512363126645770" + def href = "https://zj.lianjia.com/ershoufang/103120144204.html" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "sell_anjuke") + sendMq(href, doc, "sell_lianjia") // for(def i=25;i<50;i++){ // def url = "https://fz.anjuke.com/sale/jinana/p${i}/" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy index 19598718f..adfdce485 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy @@ -10,7 +10,7 @@ import com.mongodb.DBCollection class AnjukeSendMq { - static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_test", "192.168.0.235", 27018) + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_2207", "192.168.0.235", 27018) static main(args) { def task = new AnjukeSendMq(); @@ -19,8 +19,12 @@ class AnjukeSendMq { def run(){ def q = new BasicDBObject() - def dbcursor = collection.find(q).limit(30) + q.put("city","福州") + def dbcursor = collection.find(q).batchSize(1000) + def index = 0 for(def d : dbcursor){ + index++ + println index sendMq(d.url, d.content, "community_anjuke") } } @@ -35,6 +39,6 @@ class AnjukeSendMq { json.put("result", result) byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) + CommonUtils.sendMessage("result2parse", packedMsg) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy index a56e2079d..1fcb6a4eb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl class RegionPriceExport { @@ -18,25 +19,25 @@ class RegionPriceExport { } def run(){ - def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] - def regions = CommonUtils.getAllRegions("佛山","region") + def months = ["20220101","20220201","20220301","20220401","20220501","20220601"] + def regions = CommonUtils.getAllRegions("重庆","region") def rets = [] - rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + rets << ["城市","区域","202201","202202","202203","202204","202205","202206"] regions.each{region,values-> def data = [] - data << "佛山" + data << "重庆" data << region for(def month :months){ - def dbo = findWeekendPrice("佛山", region, month); - if(dbo){ - data << dbo.avg_price + def price = findWeekendPrice("重庆", region, month); + if(price){ + data << price }else{ data << "" } } rets << data } - ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格") + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆区域均价数据.xlsx",rets, "价格") } def findPrice(city, region, month){ @@ -53,8 +54,17 @@ class RegionPriceExport { q.put("city", city) q.put("region", region) q.put("cat", "区域") - q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) - def dbo = REGION_STAT.findOne(q) - return dbo + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + q.put("weekend", new BasicDBObject('$gte', startDate).append('$lte', endDate)) + def dbcursor = REGION_STAT.find(q) + def prices = [] + for(def d : dbcursor){ + prices << d.avg_price + } + if(prices){ + return (prices.sum()/prices.size()).toInteger() + } + return null } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 96cd65e81..eb53dc92f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -67,12 +67,12 @@ class FangjiaClientUtils { - public static def addressStand(def address){ + public static def addressStand(def city, def address){ def jsonObj = null def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") content = new HttpRequest().newGet(url, null, "utf-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 3a572c7ad..05d1cdc37 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -71,8 +71,7 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] - //def citys = CommonDatautils.findAllCity(); + def citys = CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def code = cityCodeDatas.get(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 04cc31351..ec874a945 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -17,6 +17,7 @@ import org.openqa.selenium.By import org.openqa.selenium.WebDriver import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.support.ui.ExpectedConditions import org.openqa.selenium.support.ui.WebDriverWait @@ -44,6 +45,7 @@ public class BeiKeCrawlByChrome { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) citys = citys.reverse() @@ -145,7 +147,12 @@ public class BeiKeCrawlByChrome { def detailUrls = [] while(!stop){ def url = cityUrl + "/chengjiao/pg${page}/" - driver.get(url) + try{ + driver.get(url) + }catch(Exception e){ + println url +" 出错" + } + def doc = Jsoup.parse(driver.getPageSource()); def elements = JQueryUtils.select(".listContent li", doc) if(elements){ @@ -153,7 +160,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -190,10 +197,16 @@ public class BeiKeCrawlByChrome { def roomStr = arrs[1] if (roomStr) { if (roomStr.indexOf("室") > 0) { - detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + roomStr = roomStr.substring(0, roomStr.indexOf("室")) + if(roomStr && roomStr.isNumber()){ + detail.room = roomStr.toInteger() + } } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + def hallStr = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hallStr && hallStr.isNumber()){ + detail.hall = hallStr.toInteger() + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy index 0cd120438..bf3e50a92 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -15,28 +15,29 @@ class ExportPaimai { static main(args) { def task = new ExportPaimai(); - task.run() +// task.run() // task.addressStandard() // task.addPrice() -// task.stat() + task.stat() +// task.standard() } def stat(){ - def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx" + def file = "D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已成交).xlsx" def datas = ExcelUtils.read(file, 0, 1) def index1 = 0 def rates1 = [] for(def d : datas){ if(d['P'] && d['P']=='已成交'){ - if(d['M'] && d['M']=="住宅") -// if(d['K']>=150){ +// if(d['M'] && d['M']=="住宅") +// if(d['K']>=240){ // index1++ // def sumTradePrice = d['S'] // def marketPrice = d['Y'] // rates1 << sumTradePrice/marketPrice // } - if(d['S']>2000000){ + if(d['S']<800000){ index1++ def sumTradePrice = d['S'] def marketPrice = d['Y'] @@ -56,12 +57,55 @@ class ExportPaimai { }) } def addressStandard(){ - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\需要重跑地址解析和匹配_重庆法拍数据.xlsx",{d-> def formatAddress = AddressUtil.formatAddress(d['C'], d['A']) if(formatAddress){ d['F'] = formatAddress } }) + + } + + def standard(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\需要重跑地址解析和匹配_重庆法拍数据.xlsx",{d-> + def obj = com.fangjia.script.gj.district.FangjiaClientUtils.addressStand(d['C'], d['B']) + if(obj){ + def arrs = obj.entities + def address = "" + def houseNum = "" + def roomNum = "" + def unitNum = "" + def poi = "" + for(def m : arrs){ + if(m.type == "road"){ + address = m.word + } + if(m.type == "roadno"){ + address = address + m.word + } + if(m.type == "houseno"){ + houseNum = m.word + } + if(m.type == "roomno"){ + roomNum = m.word + } + if(m.type == "cellno"){ + unitNum = m.word + } + + if(m.type == "poi"){ + poi = m.word + } + + } + + d['F'] = address + d['G'] = houseNum + d['H'] = unitNum + d['I'] = roomNum + d['J'] = poi + } + }) } def run(){ @@ -123,17 +167,20 @@ class ExportPaimai { row << d.url rows << row + }else{ + println d.title } } - ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据") + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已流拍).xlsx",rows,"数据") } def findPaimaiData(){ def pmquery = new BasicDBObject() - pmquery.put("city","重庆") + pmquery.put("city","苏州") + pmquery.put("region","姑苏") pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) - pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01"))) - pmquery.put("status","已成交") + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01"))) + pmquery.put("status","已流拍") def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1)) def rets = [] for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index ebe6f911a..0388e7fe0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome { def standardRegion(){ def q = new BasicDBObject() - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-03-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) def dbcursor = DEAL_PRICE_COLL.find(q) for(def d : dbcursor){ @@ -153,7 +153,7 @@ public class LianjiaCrawlByChrome { lastMonth = lastMonth +"-01" def q = new BasicDBObject() q.put("region", null) - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] @@ -171,7 +171,7 @@ public class LianjiaCrawlByChrome { query.put("region", null) query.put("district_name", m.name) // query.put("source", "链家在线") - query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) +// query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) def data = new BasicDBObject() data.put("region", dist.region) @@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -442,11 +442,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["福州"] + def citys= ["安庆"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 2f0d142d2..c31879151 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -23,28 +23,29 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p //String str = format.format(date); //System.out.println(str); -def pmquery = new BasicDBObject() -pmquery.put("city","宿迁") -pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) -//pmquery.put("status","已成交") -//pmquery.put("estateType", null) -pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") - -def cur = paimai43_COLL.find(pmquery) -def urls = [] -for(def o : cur){ - urls << o.url -} -println "待解析:"+urls +//def pmquery = new BasicDBObject() +//pmquery.put("city","宿迁") +//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) +////pmquery.put("status","已成交") +////pmquery.put("estateType", null) +//pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") +// +//def cur = paimai43_COLL.find(pmquery) +//def urls = [] +//for(def o : cur){ +// urls << o.url +//} +//println "待解析:"+urls def index = 0 -for(def url : urls){ +//for(def url : urls){ def q = new BasicDBObject() - q.put("url", url) +// q.put("url", url) + q.put("update_time", new BasicDBObject('$gte',DateUtils.parseFromTightByPattern("2022-07-12", "yyyy-MM-dd"))) def dbcursor = collection.find(q) - index++ - println index - for(def d : dbcursor){ + for(def d : dbcursor){ + index++ + println index JSON json = new JSONObject() json.put("project", "paimai_taobao") json.put("url", d.url) @@ -64,8 +65,8 @@ for(def url : urls){ json.put("result", result) byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) + CommonUtils.sendMessage("result2parse", packedMsg) } -} +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index a7523830b..9a8e0c117 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -34,7 +34,39 @@ class PaimaiTest { // task.updatePaimai() // task.updateFieldPaimai() // task.statCity() - task.jsStat() +// task.jsStat() + task.selectData() + } + + def selectData(){ + def urls = ["https://sf-item.taobao.com/sf_item/674045056937.htm", + "https://sf-item.taobao.com/sf_item/669281206202.htm", + "https://sf-item.taobao.com/sf_item/646818853936.htm", + "https://sf-item.taobao.com/sf_item/636815760791.htm", + "https://sf-item.taobao.com/sf_item/636055429237.htm", + "https://sf-item.taobao.com/sf_item/635094754807.htm", + "https://sf-item.taobao.com/sf_item/621286459023.htm", + "https://sf-item.taobao.com/sf_item/613944729986.htm", + "https://sf-item.taobao.com/sf_item/613944489517.htm", + "https://sf-item.taobao.com/sf_item/604967216580.htm", + "https://sf-item.taobao.com/sf_item/596302571187.htm", + "https://sf-item.taobao.com/sf_item/594722220602.htm" + ] + for(def url : urls){ + def q = new BasicDBObject() + q.put("url", url) + def obj = paimai43_COLL.findOne(q) + if(obj){ + if(obj.matchName=="中渝国宾城"){ + def udata = new BasicDBObject() + udata.put("matchName", "中渝山顶道国宾城") + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata)) + } + println url +" "+obj.matchName + }else{ + println url +" 不存在" + } + } } def jsStat(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 728e6510c..09db88159 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,17 +1,25 @@ package com.fangjia.script.gj.test +import ch.qos.logback.core.db.dialect.DBUtil import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo import groovy.sql.Sql +import org.bson.types.ObjectId import java.text.SimpleDateFormat + +def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164") +DBUtils.seaweed.save(dist) +//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist)) //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); //print dateFormat.format(date); @@ -50,13 +58,13 @@ import java.text.SimpleDateFormat //} -def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) -def rets = [] -for(def d : dists){ - def year = CommonUtils.getDistrictAge(d) - rets << [d.city,d.region, d.name, year] -} -ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") +//def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) +//def rets = [] +//for(def d : dists){ +// def year = CommonUtils.getDistrictAge(d) +// rets << [d.city,d.region, d.name, year] +//} +//ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 1b7bbc303..6ff3f087f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "Q房"// - def citys = findCrawlCity(source, 202204); + def source = "城市房产"// + def citys = findCrawlCity(source, 202205); for(def city : citys){ - def count1 = findCount(source, city, 202204); - def count2 = findCount(source, city, 202203); + def count1 = findCount(source, city, 202205); + def count2 = findCount(source, city, 202206); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 485c84e2b..7f0a598db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -193,7 +193,7 @@ class Esf58Crawl extends BaseBatch{ //addContent("福州","https://fz.58.com/xiaoqu/", 70); // "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州", - def citys = ["宁波","大连","佛山","乌鲁木齐","济南","长春","兰州","石家庄","温州","烟台","太原","泉州","南通","呼和浩特","徐州","惠州","厦门","南昌","临沂","贵阳","嘉兴","常州","盐城","威海","淄博","深圳","中山","重庆","苏州","西宁","绍兴","洛阳","廊坊","昆明","金华","珠海","宜昌","南充","信阳","杭州","沧州","宿迁","南阳","德州","桂林","新乡","包头","遵义","乐山","株洲","德阳","眉山","驻马店","许昌","宜宾","衡水","泸州","营口","清远","达州","宝鸡","长沙","广安"] + def citys = ["福州"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a95c6332f..c09e1e516 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -21,7 +21,9 @@ class SynchDistrictPrice { // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - def esSourceMap = ["ke.com":"贝壳"] + + //,"城市房产":"城市房产" + def esSources = ["365淘房":"365淘房"]//"贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客PC端":"安居客","搜房":"搜房","吉屋":"吉屋" static main(args) { def task = new SynchDistrictPrice() // task.run() @@ -31,20 +33,20 @@ class SynchDistrictPrice { def run1(){ def citys = CommonUtils.findAllCity() - esSourceMap.each { k, v -> + esSources.each{esSource, mgSource-> for(def city : citys){ - def rets = findAllEsData(city, v) + def rets = findAllEsData(city, esSource, mgSource) println city +" "+rets.size() - synchData(city, rets, "贝壳找房") + synchData(city, rets, mgSource) } } } - def findAllEsData(city,source){ + def findAllEsData(city,esSource, mgSource){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='贝壳' and city='${city}'", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-06-15'", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s @@ -54,8 +56,11 @@ class SynchDistrictPrice { data.put("name", d.name) data.put("address", d.address) data.put("avg_price", d.avg_price) - data.put("source", source) + data.put("source", mgSource) data.put("s_date", SHORT_DATE) + data.put("url", d.url) + data.put("c_date", new Date()) + data.put("m_date", new Date()) rets << data } }) @@ -138,7 +143,7 @@ class SynchDistrictPrice { def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) udata.put("m_date",new Date()) - thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata) + thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) }else{ try{