diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportNearestAvgPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportNearestAvgPrice.groovy index 62b08004f..4d1847be2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportNearestAvgPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportNearestAvgPrice.groovy @@ -29,9 +29,9 @@ public class ExportNearestAvgPrice { static main(args) { - def citys = ["苏州"]//findAllCity(); + def citys = ["重庆"]//findAllCity(); def task = new ExportNearestAvgPrice(); - def weekends = task.findWeekends(34);//日期最近的X期 + def weekends = task.findWeekends(12);//日期最近的X期 // task.exportAllCity(citys, weekends, "e:\\全国城市均价.csv"); task.exportRegion(citys, weekends, "D:\\数据\\区域均价.csv"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index c1ea41780..da60a19f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -60,39 +60,39 @@ class BigdataDealBatch extends BaseBatch{ addDealPriceMatch(city); // } } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - for(def city : citys){ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - - def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成" +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// for(def city : citys){ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// +// def trades = getCityData(city, statMap, ppeDists) +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 724682101..7088dea02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220518, 20220619); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) @@ -454,6 +454,11 @@ class BigdataForsaleBatch extends BaseBatch{ println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type continue; } + + if(seawater.estate_type&& seawater.estate_type.contains("别墅")){ + println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type + continue; + } } if(!seawater.total_price){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index fc73e4867..ee37a0616 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","3月总量","4月总量","5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-03-01" - def date = "2022-04-01" - def endDate = "2022-05-01" - def newDate = "2022-06-01" + def startDate = "2022-04-01" + def date = "2022-05-01" + def endDate = "2022-06-01" + def newDate = "2022-07-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-05成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy new file mode 100644 index 000000000..509e35e88 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocSampleTest.groovy @@ -0,0 +1,138 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class PocSampleTest { + + static main(args) { + def task = new PocSampleTest() + task.run() + } + + def run(){ +// match("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") +// addPrice("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") +// addArroundPrice("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") +// addRentRate("D:\\数据\\重庆农商行\\poc\\抽样_1000.xlsx") + +// matchResult("D:\\数据\\美团\\抽样_1000_结果.xlsx") + existsDist("D:\\数据\\美团\\抽样_1000_结果.xlsx") + } + + def existsDist(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['P']){ + def dist = CommonDatautils.findDistrictById(row['P']) + if(!dist){ + row['S'] = "不存在" + } + } + }) + } + + def matchResult(path){ + def datas = ExcelUtils.read("D:\\数据\\美团\\美团金融小区测试list-所有数据-优化租金评级.xlsx", 0 ,1) + def map = [:] + for(def r : datas){ + map.put(r['D'], r) + } + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def d = map.get(row['A']) + if(d){ + def obj = CommonDatautils.findDistrict(d['F'],d['G'],d['H'],"district") + if(obj){ + row['P'] = obj._id.toString() + row['Q'] = d['H'] + row['R'] = d['I'] + } + } + } + }) + + } + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['G'] && row['D']){ + def city = row['D'] + def region = row['E'] + if(region && region=='NULL'){ + region = row['F'] + } + def address = row['C'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, row['B'], null, true, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['H'] = jsonObject.get("name") + row['G'] = jsonObject.get("id") + row['I'] = jsonObject.get("address") + row['L'] = "地图匹配(名称当地址)" + row['M'] = jsonObject.get("propertyType") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['G'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// row['J'] = priceObj.avg_price +// } + + def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(dist.city, dist.region, dist.name) + if(sqmetreRentPrice){ + row['K'] = sqmetreRentPrice + } + } + } + + + }) + } + + def addArroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['J']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['D'], row['C']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['J'] = priceObj.unitPrice + row['N'] = "周边均价" + } + } + }); + } + + def addRentRate(path){ + def citys = ["东莞","苏州","郑州","长沙"] + def radioMap = [:] + for(def city : citys) { + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) + radioMap.put(city, avg_radio) + } + + ExcelUtils.multiThreadingModify(path, { row -> + def city = row['D'] + if (city && row['J'] && !row['K']) { + city = city.replace("市","") + def avg_rent_radio = radioMap.get(city) + if(avg_rent_radio){ + row['K'] = (int)(row['J']/avg_rent_radio) + row['O'] = "租金推算" + } + } + }) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 38d9b23b3..fede45dfc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -877,6 +877,8 @@ class CommonDatautils { keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) keys.put("sid", 1) + keys.put("status", 1) + try{ DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ @@ -2278,6 +2280,24 @@ class CommonDatautils { return prices; } + public static def queryDistrictPrices(city, region, startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))); + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + def prices = []; + while(cur.hasNext()) { + DBObject dbo = cur.next() + def price = dbo.get("avg_price"); + if(price){ + prices << price; + } + } + + return prices; + } + public static def queryDistrictPricesByDate(city, region, name, startDate, endDate){ DBObject queryCondition = new BasicDBObject() @@ -3031,9 +3051,26 @@ class CommonDatautils { return price; } + static def queryCityRegionPrices(city, region, startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("block", null); + queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))); + DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def prices = []; + while(cur.hasNext()) { + DBObject dbo = cur.next() + prices << dbo.get("avg_price"); + } + return prices; + } + + + - static def queryCityRegionBlockPrice(city, region, block, startDate, endDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index ce903527a..dfd414504 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -67,10 +67,22 @@ class DistrictInfoExportBatch extends BaseBatch{ def createDatas(def city){ def result = [] + def distKeys = [] def districts = CommonDatautils.findAllDistricts(city, "district", null); + for(def dist : districts){ + def key = dist.city +"##"+dist.region+"##"+dist.name + if(!distKeys.contains(key)){ + distKeys << key + } + } def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - districts.addAll(removeDists) - + for(def dist : removeDists){ + def key = dist.city +"##"+dist.region+"##"+dist.name + if(!distKeys.contains(key)){ + distKeys << key + districts << dist + } + } def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def startDate = DateUtils.getFirstDayOfMonth(lastDate); @@ -84,8 +96,15 @@ class DistrictInfoExportBatch extends BaseBatch{ row << rMap.city row << rMap.region row << rMap.name - row << DataHandleUtils.getDoubleRound(6, rMap.lng/10000000000d) - row << DataHandleUtils.getDoubleRound(6, rMap.lat/10000000000d) + if(rMap.lng){ + row << DataHandleUtils.getDoubleRound(6, rMap.lng/10000000000d) + row << DataHandleUtils.getDoubleRound(6, rMap.lat/10000000000d) + }else{ + row << "" + row << "" + println rMap.name +" 无经纬度" + } + row << rMap.estate_type2 row << rMap.build_type row << rMap.household_count diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index dd9cd246b..c38bb7346 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -129,7 +129,7 @@ class DistrictPriceExportBatch extends BaseBatch{ if(curPrice){ m << dbo.get("_id").toString()//均价ID }else{ - m << "" + m << new ObjectId().toString() } m << rMap.get("_id").toString()//楼盘ID m << rMap.city @@ -210,6 +210,7 @@ class DistrictPriceExportBatch extends BaseBatch{ }else{ def udata = new BasicDBObject() def o = new BasicDBObject() + println r[0] o.put("_id", new ObjectId(r[0])) o.put("comm_id", r[1]) o.put("city", r[2]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 81c5624d1..778312b6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -55,11 +56,17 @@ class FzDataResultStat { List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def distkeys = [] def map1 = [:] for(def s : list1){ def row = s.split("@\\|@") def key = row[2]+"##"+row[3]+"##"+row[4] - map1.put(key, row[5]) + if(!distkeys.contains(key)){ + distkeys << key + }else{ + println "重复:"+key + } + map1.put(key, row) } List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) @@ -67,19 +74,29 @@ class FzDataResultStat { for(def s : list2) { def row = s.split("@\\|@") def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row[5]) + map2.put(key, row) } def rows = [] - rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] - map1.each {k,v-> + rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] + map1.each {k,list-> def row = [] def keys = k.split("##") - def lastPrice = map2.get(k) - def markup = null - if(v && lastPrice){ - markup = (v.toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + + def lastObj = map2.get(k) + def lastPrice = null + if(lastObj){ + lastPrice = lastObj[5] } - rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup] + def markup = null + if(list && lastPrice){ + markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + } + def dist = CommonDatautils.findDistrictById(list[1]) + def flag = "有效" + if(dist.status!=0){ + flag = "失效" + } + rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 24a397d3e..b5aa0e611 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -73,8 +73,8 @@ class HouseExportBatch extends BaseBatch{ def fileName = "/fzbank/output/FJW_HOUSE_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_HOUSE_"+curDate+".sha512" if(CommonDatautils.isWindow()){ - fileName = "F:\\数据\\福州\\FJW_HOUSE_"+curDate+".csv" - dataCodeFileName = "F:\\数据\\福州\\FJW_HOUSE_"+curDate+".sha512" + fileName = "D:\\数据\\福州\\FJW_HOUSE_"+curDate+".csv" + dataCodeFileName = "D:\\数据\\福州\\FJW_HOUSE_"+curDate+".sha512" } batchCreateFile(ret, fileName, headers, charset); println "数据文件输出完成......" @@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{ // int endDate = 20191201 def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd") - def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); - + // def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); + def endDate = "20220710" QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); // QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false); @@ -420,7 +420,8 @@ class HouseExportBatch extends BaseBatch{ row << seawater.name row << seawater.url - row << seawater.p_date +// row << seawater.p_date + row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") rets << row } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 563e6ab47..43e47801d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202205.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -136,8 +136,8 @@ def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计") def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s -//def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -//DingDingMessageUtil.sendTextMessage(access_token, s) +def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" +DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 68494ea8b..4f8e69217 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-03-01" + static def statDate = "2022-06-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo - task.esfParser();//友达二手房存入mongo +// task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年7月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index 21ddf507c..b97ec6bb1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -44,7 +44,7 @@ class AnjukeAppCrawl { // String line2 = StringEscapeUtils.unescapeJava(s); // println line2 def task = new AnjukeAppCrawl(); -// task.autoRun() + task.autoRun() // task.run(); // task.run2() @@ -55,7 +55,7 @@ class AnjukeAppCrawl { // task.groupByCity() // task.selectNoSave() // task.statCount() - task.exportFzDist() +// task.exportFzDist() } def saveToEs1(){ def lists = getAllAppDatas() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy new file mode 100644 index 000000000..afa1280a4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + +class AnjukeDistSearch { + static main(args) { + def task = new AnjukeDistSearch() + task.run() + } + + def run(){ + def file = "D:\\数据\\朔州匹配房源1.xlsx" + ExcelUtils.modify(file,{d-> + def data = getDataInfo(282, d['C']) + if(data){ + def match = data.match + if(match){ + println match + } + } + }) + getDataInfo() + } + + def getDataInfo(cityId, keyword){ + def url = "https://m.anjuke.com/ajax/autocomplete/?city_id=282&kw="+URLEncoder.encode(keyword, "utf-8") + def ret = ABuYunUtil.doGet(url, null, null, "utf-8") + if (ret) { + JSONObject jsonObj = JSONObject.fromObject(ret); + if (jsonObj && jsonObj.msg == "ok") { + def data = jsonObj.data + return data + } + } + return null + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index bd7567c8e..065010b5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,11 +25,11 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://fz.anjuke.com/prop/view/S2512363126645770" + def href = "https://zj.lianjia.com/ershoufang/103120144204.html" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "sell_anjuke") + sendMq(href, doc, "sell_lianjia") // for(def i=25;i<50;i++){ // def url = "https://fz.anjuke.com/sale/jinana/p${i}/" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy index 19598718f..adfdce485 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy @@ -10,7 +10,7 @@ import com.mongodb.DBCollection class AnjukeSendMq { - static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_test", "192.168.0.235", 27018) + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_2207", "192.168.0.235", 27018) static main(args) { def task = new AnjukeSendMq(); @@ -19,8 +19,12 @@ class AnjukeSendMq { def run(){ def q = new BasicDBObject() - def dbcursor = collection.find(q).limit(30) + q.put("city","福州") + def dbcursor = collection.find(q).batchSize(1000) + def index = 0 for(def d : dbcursor){ + index++ + println index sendMq(d.url, d.content, "community_anjuke") } } @@ -35,6 +39,6 @@ class AnjukeSendMq { json.put("result", result) byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) + CommonUtils.sendMessage("result2parse", packedMsg) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy index a56e2079d..1fcb6a4eb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl class RegionPriceExport { @@ -18,25 +19,25 @@ class RegionPriceExport { } def run(){ - def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] - def regions = CommonUtils.getAllRegions("佛山","region") + def months = ["20220101","20220201","20220301","20220401","20220501","20220601"] + def regions = CommonUtils.getAllRegions("重庆","region") def rets = [] - rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"] + rets << ["城市","区域","202201","202202","202203","202204","202205","202206"] regions.each{region,values-> def data = [] - data << "佛山" + data << "重庆" data << region for(def month :months){ - def dbo = findWeekendPrice("佛山", region, month); - if(dbo){ - data << dbo.avg_price + def price = findWeekendPrice("重庆", region, month); + if(price){ + data << price }else{ data << "" } } rets << data } - ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格") + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆区域均价数据.xlsx",rets, "价格") } def findPrice(city, region, month){ @@ -53,8 +54,17 @@ class RegionPriceExport { q.put("city", city) q.put("region", region) q.put("cat", "区域") - q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) - def dbo = REGION_STAT.findOne(q) - return dbo + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(month, "yyyyMMdd")) + q.put("weekend", new BasicDBObject('$gte', startDate).append('$lte', endDate)) + def dbcursor = REGION_STAT.find(q) + def prices = [] + for(def d : dbcursor){ + prices << d.avg_price + } + if(prices){ + return (prices.sum()/prices.size()).toInteger() + } + return null } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 96cd65e81..eb53dc92f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -67,12 +67,12 @@ class FangjiaClientUtils { - public static def addressStand(def address){ + public static def addressStand(def city, def address){ def jsonObj = null def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") content = new HttpRequest().newGet(url, null, "utf-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 3a572c7ad..05d1cdc37 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -71,8 +71,7 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] - //def citys = CommonDatautils.findAllCity(); + def citys = CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def code = cityCodeDatas.get(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 04cc31351..ec874a945 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -17,6 +17,7 @@ import org.openqa.selenium.By import org.openqa.selenium.WebDriver import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.support.ui.ExpectedConditions import org.openqa.selenium.support.ui.WebDriverWait @@ -44,6 +45,7 @@ public class BeiKeCrawlByChrome { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) citys = citys.reverse() @@ -145,7 +147,12 @@ public class BeiKeCrawlByChrome { def detailUrls = [] while(!stop){ def url = cityUrl + "/chengjiao/pg${page}/" - driver.get(url) + try{ + driver.get(url) + }catch(Exception e){ + println url +" 出错" + } + def doc = Jsoup.parse(driver.getPageSource()); def elements = JQueryUtils.select(".listContent li", doc) if(elements){ @@ -153,7 +160,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -190,10 +197,16 @@ public class BeiKeCrawlByChrome { def roomStr = arrs[1] if (roomStr) { if (roomStr.indexOf("室") > 0) { - detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + roomStr = roomStr.substring(0, roomStr.indexOf("室")) + if(roomStr && roomStr.isNumber()){ + detail.room = roomStr.toInteger() + } } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + def hallStr = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hallStr && hallStr.isNumber()){ + detail.hall = hallStr.toInteger() + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy index 0cd120438..bf3e50a92 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -15,28 +15,29 @@ class ExportPaimai { static main(args) { def task = new ExportPaimai(); - task.run() +// task.run() // task.addressStandard() // task.addPrice() -// task.stat() + task.stat() +// task.standard() } def stat(){ - def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx" + def file = "D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已成交).xlsx" def datas = ExcelUtils.read(file, 0, 1) def index1 = 0 def rates1 = [] for(def d : datas){ if(d['P'] && d['P']=='已成交'){ - if(d['M'] && d['M']=="住宅") -// if(d['K']>=150){ +// if(d['M'] && d['M']=="住宅") +// if(d['K']>=240){ // index1++ // def sumTradePrice = d['S'] // def marketPrice = d['Y'] // rates1 << sumTradePrice/marketPrice // } - if(d['S']>2000000){ + if(d['S']<800000){ index1++ def sumTradePrice = d['S'] def marketPrice = d['Y'] @@ -56,12 +57,55 @@ class ExportPaimai { }) } def addressStandard(){ - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\需要重跑地址解析和匹配_重庆法拍数据.xlsx",{d-> def formatAddress = AddressUtil.formatAddress(d['C'], d['A']) if(formatAddress){ d['F'] = formatAddress } }) + + } + + def standard(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\需要重跑地址解析和匹配_重庆法拍数据.xlsx",{d-> + def obj = com.fangjia.script.gj.district.FangjiaClientUtils.addressStand(d['C'], d['B']) + if(obj){ + def arrs = obj.entities + def address = "" + def houseNum = "" + def roomNum = "" + def unitNum = "" + def poi = "" + for(def m : arrs){ + if(m.type == "road"){ + address = m.word + } + if(m.type == "roadno"){ + address = address + m.word + } + if(m.type == "houseno"){ + houseNum = m.word + } + if(m.type == "roomno"){ + roomNum = m.word + } + if(m.type == "cellno"){ + unitNum = m.word + } + + if(m.type == "poi"){ + poi = m.word + } + + } + + d['F'] = address + d['G'] = houseNum + d['H'] = unitNum + d['I'] = roomNum + d['J'] = poi + } + }) } def run(){ @@ -123,17 +167,20 @@ class ExportPaimai { row << d.url rows << row + }else{ + println d.title } } - ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据") + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已流拍).xlsx",rows,"数据") } def findPaimaiData(){ def pmquery = new BasicDBObject() - pmquery.put("city","重庆") + pmquery.put("city","苏州") + pmquery.put("region","姑苏") pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) - pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01"))) - pmquery.put("status","已成交") + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01"))) + pmquery.put("status","已流拍") def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1)) def rets = [] for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index ebe6f911a..0388e7fe0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome { def standardRegion(){ def q = new BasicDBObject() - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-03-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) def dbcursor = DEAL_PRICE_COLL.find(q) for(def d : dbcursor){ @@ -153,7 +153,7 @@ public class LianjiaCrawlByChrome { lastMonth = lastMonth +"-01" def q = new BasicDBObject() q.put("region", null) - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] @@ -171,7 +171,7 @@ public class LianjiaCrawlByChrome { query.put("region", null) query.put("district_name", m.name) // query.put("source", "链家在线") - query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) +// query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) def data = new BasicDBObject() data.put("region", dist.region) @@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -442,11 +442,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["福州"] + def citys= ["安庆"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 2f0d142d2..c31879151 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -23,28 +23,29 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p //String str = format.format(date); //System.out.println(str); -def pmquery = new BasicDBObject() -pmquery.put("city","宿迁") -pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) -//pmquery.put("status","已成交") -//pmquery.put("estateType", null) -pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") - -def cur = paimai43_COLL.find(pmquery) -def urls = [] -for(def o : cur){ - urls << o.url -} -println "待解析:"+urls +//def pmquery = new BasicDBObject() +//pmquery.put("city","宿迁") +//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) +////pmquery.put("status","已成交") +////pmquery.put("estateType", null) +//pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") +// +//def cur = paimai43_COLL.find(pmquery) +//def urls = [] +//for(def o : cur){ +// urls << o.url +//} +//println "待解析:"+urls def index = 0 -for(def url : urls){ +//for(def url : urls){ def q = new BasicDBObject() - q.put("url", url) +// q.put("url", url) + q.put("update_time", new BasicDBObject('$gte',DateUtils.parseFromTightByPattern("2022-07-12", "yyyy-MM-dd"))) def dbcursor = collection.find(q) - index++ - println index - for(def d : dbcursor){ + for(def d : dbcursor){ + index++ + println index JSON json = new JSONObject() json.put("project", "paimai_taobao") json.put("url", d.url) @@ -64,8 +65,8 @@ for(def url : urls){ json.put("result", result) byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) + CommonUtils.sendMessage("result2parse", packedMsg) } -} +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index a7523830b..9a8e0c117 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -34,7 +34,39 @@ class PaimaiTest { // task.updatePaimai() // task.updateFieldPaimai() // task.statCity() - task.jsStat() +// task.jsStat() + task.selectData() + } + + def selectData(){ + def urls = ["https://sf-item.taobao.com/sf_item/674045056937.htm", + "https://sf-item.taobao.com/sf_item/669281206202.htm", + "https://sf-item.taobao.com/sf_item/646818853936.htm", + "https://sf-item.taobao.com/sf_item/636815760791.htm", + "https://sf-item.taobao.com/sf_item/636055429237.htm", + "https://sf-item.taobao.com/sf_item/635094754807.htm", + "https://sf-item.taobao.com/sf_item/621286459023.htm", + "https://sf-item.taobao.com/sf_item/613944729986.htm", + "https://sf-item.taobao.com/sf_item/613944489517.htm", + "https://sf-item.taobao.com/sf_item/604967216580.htm", + "https://sf-item.taobao.com/sf_item/596302571187.htm", + "https://sf-item.taobao.com/sf_item/594722220602.htm" + ] + for(def url : urls){ + def q = new BasicDBObject() + q.put("url", url) + def obj = paimai43_COLL.findOne(q) + if(obj){ + if(obj.matchName=="中渝国宾城"){ + def udata = new BasicDBObject() + udata.put("matchName", "中渝山顶道国宾城") + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata)) + } + println url +" "+obj.matchName + }else{ + println url +" 不存在" + } + } } def jsStat(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy new file mode 100644 index 000000000..fdc7e3197 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy @@ -0,0 +1,275 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) +DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) +DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + +DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43") + +//def q = new BasicDBObject() +//q.put("city","重庆") +//q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))) +//def dbcursor = DEAL_PRICE_COLL.find(q) +//def map = [:] +//for(def d : dbcursor){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } +// def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd") +// if(!list.contains(key)){ +// list << key +// } +//} + + +//def q = new BasicDBObject() +//q.put("status","已成交") +//q.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))) +//def dbcursor = paimai43_COLL.find(q) +//def map = [:] +//for(def d : dbcursor){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } +// if(!list.contains(d.title)){ +// list << d.title +// } +//} + + +//ExcelUtils.modify("D:\\数据\\朔州房源.xlsx",{r-> +// def dist = CommonDatautils.findDistrict(r['A'],r['B'], r['C'], "district") +// if(dist){ +// r['E'] = "名称完全一致" +// } +// +//}) + +def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"] + +def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"]) + +def totalCount = new AtomicLong() +final CountDownLatch latch = new CountDownLatch(districts.size()) +println "小区量:"+districts.size() + +def retMap = new ConcurrentHashMap() +for(def i=0;i +// def arrs = k.split("#") +// datas << [arrs[0], arrs[1], arrs[2], count] +//} +//ExcelUtils.write("D:\\数据\\朔州匹配房源1.xlsx", datas, "数量") +//ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆数据统计.xlsx",{d-> +// def count = retMap.get(d['A']) +// if(count){ +// def pre = count.toInteger()*100/d['B'].toString().toDouble().toInteger() +// pre = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, pre) +// d['E'] = pre +// } +// if(d['A'] && !d['H']){ +// def prices = CommonDatautils.queryCityRegionPrices("重庆", d['A'], '2022-01-01') +// if(prices){ +// def avg_prices = prices.sum()/prices.size() +// d['H'] = avg_prices.toInteger() +// } +// def prices = CommonDatautils.queryDistrictPrices("重庆", d['A'], '2022-01-01') +// if(prices){ +// def avg_prices = prices.sum()/prices.size() +// d['H'] = avg_prices.toInteger() +// } +// } +//}) + + +//ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d-> +// def rets = findHousesByCityAndRoot(d['A'], "sell", 20210701, "anjuke.com") +// d['R'] = rets.size() +// println d['A'] +" "+rets.size() +// +//}) + +def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058] +ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d-> + def count = countMap.get(d['A']) + d['R'] = count +}) + +println "完成" + + +def findThirdParty(city, region, name, coll){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', 202107)) + def dbcursor = coll.find(q) + def rets = [] + for(def d : dbcursor){ + if(d && d.size()>9){ + println d + rets << d + } + } + return rets +} + + +def findHouses(def city, def region, def cat, def startDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; +} + + +def findHousesByCity(def city, def cat, def startDate){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; +} + +def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range).must(QueryBuilders.termQuery("root", root)) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; +} + +//def map = [:] +//for(def d : dists){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } +// list << d +//} +// +//def rows = [] +//rows << ["区域","小区量", "别名量", "地址别名量"] +//map.each{region, list-> +// def alias_count = 0 +// def addr_alias_count = 0 +// for(def r : list){ +// if(r.mapping_alias && r.mapping_alias.size()>0){ +// alias_count ++ +// } +// if(r.addr_alias && r.addr_alias.size()>0){ +// addr_alias_count++ +// } +// } +// +// rows << [region, list.size(), alias_count, addr_alias_count] +//} +// +//ExcelUtils.write("D:\\数据\\重庆农商行\\重庆数据统计.xlsx", rows, "楼盘统计") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 728e6510c..09db88159 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,17 +1,25 @@ package com.fangjia.script.gj.test +import ch.qos.logback.core.db.dialect.DBUtil import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo import groovy.sql.Sql +import org.bson.types.ObjectId import java.text.SimpleDateFormat + +def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164") +DBUtils.seaweed.save(dist) +//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist)) //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); //print dateFormat.format(date); @@ -50,13 +58,13 @@ import java.text.SimpleDateFormat //} -def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) -def rets = [] -for(def d : dists){ - def year = CommonUtils.getDistrictAge(d) - rets << [d.city,d.region, d.name, year] -} -ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") +//def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) +//def rets = [] +//for(def d : dists){ +// def year = CommonUtils.getDistrictAge(d) +// rets << [d.city,d.region, d.name, year] +//} +//ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index af5f5fc8b..2a2972c21 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -24,10 +24,6 @@ import java.util.concurrent.atomic.AtomicInteger */ class GaoDeUtils { - // 新获取key - def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"] - def static AtomicInteger keyUsed = new AtomicInteger(0) - def static codeNameMap = new HashMap(){{ put("150500", "地铁"); put("1507", "公交"); @@ -54,6 +50,20 @@ class GaoDeUtils { put("0601", "商场") }}; + /** + * 获取高德KEY + * @return + */ + static String getGaodeKey(String type) { + Map params = new HashMap<>(); + params.put("type", type) + JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params) + if (json.status == 0 && json.result != null) { + return json.result + } + System.exit(1); + } + /** * 查找周边配套 * @param city @@ -66,7 +76,7 @@ class GaoDeUtils { */ static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] - params.key = GAO_DE_AK[keyUsed] + if (types) { params.types = types @@ -83,25 +93,17 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ + params.key = this.getGaodeKey("v3/place/around") JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { + params.key = this.getGaodeKey("v3/place/around") json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() result.addAll(DataUtil.toMongoDbObj(json.pois)) } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - break; - } + System.exit(1) } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) @@ -128,7 +130,6 @@ class GaoDeUtils { */ static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] - params.key = GAO_DE_AK[keyUsed.intValue()] if (types) { params.types = types @@ -145,8 +146,10 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ + params.key = this.getGaodeKey("v5/place/around") JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); if (json == null) { + params.key = this.getGaodeKey("v5/place/around") json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); } if (json && json.info && json.info == "OK") { @@ -158,26 +161,14 @@ class GaoDeUtils { ex.printStackTrace() } } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - break; - } + System.exit(1) } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - //System.exit(1) - Thread.sleep(60000) - break; + System.exit(1) } else { break } @@ -225,11 +216,12 @@ class GaoDeUtils { */ static List getAmapById(id){ def params = [:] - params.key = GAO_DE_AK[keyUsed.intValue()] params.id = id params.output = 'json' + params.key = this.getGaodeKey("v3/place/detail") JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); if (json == null) { + params.key = this.getGaodeKey("v3/place/detail") json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); } if (json && json.info && json.info == "OK") { @@ -240,23 +232,13 @@ class GaoDeUtils { ex.printStackTrace() } } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - } + System.exit(1) } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - //System.exit(1) - Thread.sleep(60000) + System.exit(1) } return Collections.emptyList() } @@ -266,11 +248,12 @@ class GaoDeUtils { */ static List getAmapDetailById(id){ def params = [:] - params.key = GAO_DE_AK[keyUsed.intValue()] params.id = id params.output = 'json' + params.key = this.getGaodeKey("v3/place/detail") JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); if (json == null) { + params.key = this.getGaodeKey("v3/place/detail") json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); } if (json && json.info && json.info == "OK") { @@ -281,23 +264,13 @@ class GaoDeUtils { ex.printStackTrace() } } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - } + System.exit(1) } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - //System.exit(1) - Thread.sleep(60000) + System.exit(1) } return Collections.emptyList() } @@ -307,7 +280,6 @@ class GaoDeUtils { */ static List searchPoi(String city, String keywords, String region, String typecodes){ def params = [:] - params.key = GAO_DE_AK[keyUsed.intValue()] params.keywords = keywords params.city = city params.region = region @@ -324,8 +296,10 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ + params.key = this.getGaodeKey("v5/place/text") JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); if (json == null) { + params.key = this.getGaodeKey("v5/place/text") json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); } if (json && json.info && json.info == "OK") { @@ -337,17 +311,7 @@ class GaoDeUtils { ex.printStackTrace() } } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - break; - } + System.exit(1) } else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) @@ -377,12 +341,13 @@ class GaoDeUtils { */ static def getGaoDeAddress(location) { Map params = new HashMap() - params.put("key", GAO_DE_AK[keyUsed.intValue()]) params.put("location", location) String content try { + params.put("key", this.getGaodeKey("v3/geocode/regeo")) content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") } catch (Exception e) { + params.put("key", this.getGaodeKey("v3/geocode/regeo")) content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") } net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) @@ -393,16 +358,7 @@ class GaoDeUtils { return regeocode.formatted_address; } } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - } + System.exit(1) } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) @@ -418,13 +374,14 @@ class GaoDeUtils { */ static def parserByGaode(def city, def address) { Map params = new HashMap() - params.put("key", GAO_DE_AK[keyUsed.intValue()]) params.put("city", city) params.put("address", address) String content try { + params.put("key", this.getGaodeKey("v3/geocode/geo")) content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") } catch (Exception e) { + params.put("key", this.getGaodeKey("v3/geocode/geo")) content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") } net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) @@ -437,16 +394,7 @@ class GaoDeUtils { return ["lng": arrs[0], "lat": arrs[1]] } } else if (json && json.infocode && json.infocode == "10044") { - synchronized (this) { - for (int i = 0; i < GAO_DE_AK.size(); i++) { - if (GAO_DE_AK[i] == params.key) { - if (i + 1 >= GAO_DE_AK.size()) { - System.exit(1) - } - keyUsed = new AtomicInteger(i+1) - } - } - } + System.exit(1) } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) @@ -455,4 +403,8 @@ class GaoDeUtils { System.exit(1) } } + + static void main(String[] args) { + println(this.getGaodeKey("v3/geocode/geo")) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 1b7bbc303..6ff3f087f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "Q房"// - def citys = findCrawlCity(source, 202204); + def source = "城市房产"// + def citys = findCrawlCity(source, 202205); for(def city : citys){ - def count1 = findCount(source, city, 202204); - def count2 = findCount(source, city, 202203); + def count1 = findCount(source, city, 202205); + def count2 = findCount(source, city, 202206); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 485c84e2b..7f0a598db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -193,7 +193,7 @@ class Esf58Crawl extends BaseBatch{ //addContent("福州","https://fz.58.com/xiaoqu/", 70); // "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州", - def citys = ["宁波","大连","佛山","乌鲁木齐","济南","长春","兰州","石家庄","温州","烟台","太原","泉州","南通","呼和浩特","徐州","惠州","厦门","南昌","临沂","贵阳","嘉兴","常州","盐城","威海","淄博","深圳","中山","重庆","苏州","西宁","绍兴","洛阳","廊坊","昆明","金华","珠海","宜昌","南充","信阳","杭州","沧州","宿迁","南阳","德州","桂林","新乡","包头","遵义","乐山","株洲","德阳","眉山","驻马店","许昌","宜宾","衡水","泸州","营口","清远","达州","宝鸡","长沙","广安"] + def citys = ["福州"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a95c6332f..c09e1e516 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -21,7 +21,9 @@ class SynchDistrictPrice { // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - def esSourceMap = ["ke.com":"贝壳"] + + //,"城市房产":"城市房产" + def esSources = ["365淘房":"365淘房"]//"贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客PC端":"安居客","搜房":"搜房","吉屋":"吉屋" static main(args) { def task = new SynchDistrictPrice() // task.run() @@ -31,20 +33,20 @@ class SynchDistrictPrice { def run1(){ def citys = CommonUtils.findAllCity() - esSourceMap.each { k, v -> + esSources.each{esSource, mgSource-> for(def city : citys){ - def rets = findAllEsData(city, v) + def rets = findAllEsData(city, esSource, mgSource) println city +" "+rets.size() - synchData(city, rets, "贝壳找房") + synchData(city, rets, mgSource) } } } - def findAllEsData(city,source){ + def findAllEsData(city,esSource, mgSource){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='贝壳' and city='${city}'", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-06-15'", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s @@ -54,8 +56,11 @@ class SynchDistrictPrice { data.put("name", d.name) data.put("address", d.address) data.put("avg_price", d.avg_price) - data.put("source", source) + data.put("source", mgSource) data.put("s_date", SHORT_DATE) + data.put("url", d.url) + data.put("c_date", new Date()) + data.put("m_date", new Date()) rets << data } }) @@ -138,7 +143,7 @@ class SynchDistrictPrice { def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) udata.put("m_date",new Date()) - thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata) + thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) }else{ try{