diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 811ec290c..cd1282a81 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -433,11 +433,17 @@ javadbf 1.9.2 - + + + ru.yandex.clickhouse + clickhouse-jdbc + 0.2.6 + + com.alibaba fastjson diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy index e85c93bdd..fb9821df4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy @@ -30,12 +30,12 @@ public class ExportAvgPrice { DSCollection dataStoreConnection = null; static main(args) { def task = new ExportAvgPrice(); - def weekends = ["2019-1-1","2019-2-1","2019-3-1","2019-4-1","2019-5-1","2019-6-1","2019-7-1","2019-8-1", "2019-9-1","2019-10-1","2019-11-1","2019-12-1","2020-1-1","2020-2-1","2020-3-1","2020-4-1","2020-5-1","2020-6-1","2020-7-1","2020-8-1", "2020-9-1","2020-10-1","2020-11-1","2020-12-1","2021-1-1","2021-2-1","2021-3-1","2021-4-1","2021-5-1","2021-6-1","2021-7-1","2021-8-1", "2021-9-1","2021-10-1"]; - def citys = ["温州"];//findAllCity(); -// task.exportAllCity(citys, weekends, "e:\\全国城市季度均价.csv"); - task.exportRegion(citys, weekends, "d:\\温州区域季度均价.csv"); + def weekends = ["2023-04-01","2023-03-01","2023-02-01","2023-01-01","2022-12-01","2022-11-01","2022-10-01","2022-09-01","2022-08-01","2022-07-01","2022-06-01","2022-05-01","2022-04-01","2022-03-01","2022-02-01","2022-01-01","2021-12-01","2021-11-01","2021-10-01","2021-09-01","2021-08-01","2021-07-01","2021-06-01","2021-05-01","2021-04-01"] + def citys = ["上海"];//findAllCity(); +// task.exportAllCity(citys, weekends, "D:\\数据\\小区\\全国城市季度均价.csv"); +// task.exportRegion(citys, weekends, "D:\\数据\\小区\\区域季度均价.csv"); // task.exportBlock(citys, weekends, "e:\\区域板块季度均价.csv"); -// task.exportDistrict(citys, weekends, "e:\\小区季度均价.csv"); + task.exportDistrict(citys, weekends, "D:\\数据\\小区\\小区季度均价.csv"); // task.exportRegionPrice("江苏"); } @@ -182,6 +182,7 @@ public class ExportAvgPrice { q.put("visible", 0); q.put("cat", "district"); q.put("city", city); +// q.put("name", "龙柏二村") DSCursor cursor = getDataStoreConnection().find(q).sort(new BasicDBObject('bscore', -1)); Iterator iters = cursor.iterator() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy new file mode 100644 index 000000000..0a601a6ec --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1) +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + +for(def d : datas){ + def map = new BasicDBObject() + map.city = "深圳" + map.district_name = d['D'] + map.region = d['B'] + def roomStr = d['E'] + if(roomStr){ + def arrs = roomStr.split("-") + map.room = arrs[0].toString().toInteger() + map.hall = arrs[1].toString().toInteger() + map.toilet = arrs[3].toString().toInteger() + } + + map.area = d['F'].toString().toDouble() + map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) + map.avg_price = (int)(d['H'].toString().toDouble()*10000) + def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd") + map.source = "深房家族" + map.create_date = new Date() + map.m_date = new Date() + DEAL_PRICE_COLL.save(map) +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 4edbf78a5..345491bd5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) -// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 - def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 +// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt 生成->导Oracle" - delPgData() - saveToPgSql(rows) +// delPgData() +// saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index e00fdeb41..5e7d45516 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{ } println "curDate:"+curDate -// for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } -// } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 for(def city : citys){ - index ++ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成 " + rows.size() - - saveToPgSql(rows) +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// for(def city : citys){ +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +// saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 845b06548..de1e2d965 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -430,6 +430,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); + q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 // q.put("name","锦南浦三小区") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index d579b79f1..cdad2a607 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -375,12 +375,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202302 + def month = 202303 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202303) + def rets = findPeGteThirdParty2(city, 202304) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 4c9574fd8..5c2e8c26c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -33,8 +33,8 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() - task.addFjwPrice() -// task.addCcbPrice() +// task.addFjwPrice() +// task.addCcbPrice()//oracle // task.addCcbPriceFromGuided() // task.addMarketPrice() @@ -42,21 +42,36 @@ class FjwPricePkCcbPrice { // task.exportFjwPriceAndCcbPrice() // task.addNewDistPrice() -// task.addCcbPrice2() -// task.addCcbPrice3() + task.addCcbPrice2()//按id查 +// task.addCcbPrice3()//按city,region,name查 // task.standRegion() +// task.addGuidPrice() } + def addCcbPrice2(){ def cityCodeMap = CcbCommonUtils.findCityCodeValue() - ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d -> - if(d['R']){ - def map = findAvgPrice(d['R'], cityCodeMap.get(d['S'])) + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d -> + if(d['C']){ + def map = findAvgPrice(d['C'], cityCodeMap.get(d['E'])) if(map){ - d['AG'] = map.avg_price - d['AH'] = map.calc_type - d['AI'] = map.credit_value - d['AJ'] = map.month + d['K'] = map.avg_price + d['L'] = map.calc_type +// d['AI'] = map.credit_value + d['M'] = map.month + } + } + }) + } + + def addGuidPrice(){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_230421.xlsx", { d -> + if(d['M']){ + def map = findGuidePrice(d['C'], d['E']) + if(map){ + d['M'] = map.avg_price + d['N'] = map.date.toString() } } }) @@ -474,4 +489,14 @@ class FjwPricePkCcbPrice { }) return map } + + def findGuidePrice(def commid, def city){ + def sql = "SELECT gujia_bds_drc_prc,gujia_drc_dt from base.t0831_guided_price_gj_h where urbn_nm='${city}' and gujia_bds_id='${commid}' ORDER BY gujia_drc_dt desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_drc_prc + map.date = r.gujia_drc_dt + }) + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 716114623..b78a31710 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 5474a893c..db8fef0a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -71,33 +71,33 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // //}) -def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1) -def allCitys = CcbCommonUtils.citys_all -def cityCodeMap = CcbCommonUtils.findCityCodeValue() -def index = 0 -for(def c : allCitys){ - def cityCode = cityCodeMap.get(c) - def count = null; - index++ - for(def r : countCitys){ - if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){ - count = r['B'] - break - } - } - println index +" "+ c+" "+count -} - -def findCcbPriceCredit(sqlInstance, cityid, commid){ - def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" - def map = [:] - sqlInstance.eachRow(sql,{r-> - map.month = r.year_month - map.credit = r.CONFIDENCE - map.price = r.avg_price - }); - return map -} +//def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1) +//def allCitys = CcbCommonUtils.citys_all +//def cityCodeMap = CcbCommonUtils.findCityCodeValue() +//def index = 0 +//for(def c : allCitys){ +// def cityCode = cityCodeMap.get(c) +// def count = null; +// index++ +// for(def r : countCitys){ +// if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){ +// count = r['B'] +// break +// } +// } +// println index +" "+ c+" "+count +//} +// +//def findCcbPriceCredit(sqlInstance, cityid, commid){ +// def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" +// def map = [:] +// sqlInstance.eachRow(sql,{r-> +// map.month = r.year_month +// map.credit = r.CONFIDENCE +// map.price = r.avg_price +// }); +// return map +//} def findThirdPriceBySource(sqlInstance, cityid, commid){ def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" + @@ -1314,6 +1314,24 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ +def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") + if(count && count==1){ + def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") + d['E'] = dist.city + } + +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// d['B'] = dist.region +// d['C'] = dist.name +// d['J'] = dist.status +// } +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// d['F'] = newRegion +},0) + //ExcelUtils.modify(fileName, {d-> // if(d['A']){ // def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 076edf823..3c0226e81 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","12月总量","23年1月总量","23年2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-12-01" - def date = "2023-01-01" - def endDate = "2023-02-01" - def newDate = "2023-03-01" + def startDate = "2023-01-01" + def date = "2023-02-01" + def endDate = "2023-03-01" + def newDate = "2023-04-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 9ed8300cc..5a38c237e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); -// cityRun(citys, startDate, endDate, cityCodeMap) -// regionRun(citys, startDate, endDate, cityCodeMap) + cityRun(citys, startDate, endDate, cityCodeMap) + regionRun(citys, startDate, endDate, cityCodeMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index ce4355ead..43132328a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -34,6 +34,7 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") citys = CcbCommonUtils.getRunningCitys(args) @@ -76,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataModelingBase(lastMonth) //2月8号 临时导出苏州一年的数据 - exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 + exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 } if(curDate.toString().endsWith("28")){ @@ -1012,7 +1013,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportPgThirdPartyPrice2(price_date, new_dists){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -1184,7 +1185,7 @@ class ExportDistrictRongHe extends BaseBatch{ }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" @@ -1273,7 +1274,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportPgCommunityPdescribeGj(month){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" @@ -1381,7 +1382,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportPgGuideStatistics(time){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" @@ -1417,7 +1418,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportPgBigdataSupplierStatistics(time){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" @@ -1532,7 +1533,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportPgBigdataModelingBase(time){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index cae3e60b2..b7bf2feb6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -622,7 +622,10 @@ class CommonDatautils { public static def findDistrictCount(String city, String region, String name, String cat){ DBObject q = new BasicDBObject(); - q.put("city", city); + if(city){ + q.put("city", city); + } + if(region){ q.put("region", region); } @@ -639,7 +642,10 @@ class CommonDatautils { public static def findDistrict(String city, String region, String name, String cat){ DBObject q = new BasicDBObject(); - q.put("city", city); + if(city){ + q.put("city", city); + } + if(region){ q.put("region", region); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 2da506d11..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); -// task.regionPriceTrendRun() + task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 89347b833..e26dc05dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 7ed37c86c..8ccb30a9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -24,7 +24,13 @@ class AddAvgpriceBatch { static main(args) { def task = new AddAvgpriceBatch(); - task.run1(); +// task.run1(); + task.test() + } + + def test(){ + //发重算均价命令 + addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd')) } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index cdbeaefd7..805c41944 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -263,8 +264,7 @@ class FiledStrand { if(all_area){ def v = getValue(all_area) if(v>=1000 && v<=10000000){ - - return v.toString() + return v.toString().toDouble().toInteger() } } return null @@ -273,7 +273,7 @@ class FiledStrand { static def processCarSite(car_site_count){ def v = getValue(car_site_count.toString()); if(v){ - if(v>=1 && v<=100000){ + if(v>5 && v<=100000){ try { return v.toString().toInteger(); }catch(Exception e){ @@ -284,6 +284,70 @@ class FiledStrand { return null } + static def processCarSiteDesc(car_site){ + if(car_site && car_site.size()<=20){ + return car_site.toString() + } + return null + } + + + static def processPropertyLife(car_site_count){ + def v = getValue(car_site_count.toString()); + if(v){ + if([40,50,70].contains(v)){ + try { + return v.toString()+"年"; + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + + static def processAlias(esVal, name, alias){ + if(esVal){ + if(esVal!=name){ + if(!alias){ + alias = [] + } + def arr = alias.split(",") + for(def s : arr){ + if(!alias.contains(s)){ + alias << esVal + } + } + + if(alias){ + return alias + } + } + } + return null + } + + static def processAddress(def esAddress, def city){ + if(esAddress){ + def formatAddress = AddressUtil.formatAddress(city, esAddress) + if(formatAddress && isRoad(formatAddress)){ + return formatAddress + } + } + return null + } + + static def isRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + + + static def addBuildAreaToData(def dist, def updateFile){ def build_area = null; if(dist.build_area){ @@ -324,18 +388,12 @@ class FiledStrand { return updateFile; } - public static def processCubagePercent(cubage_percent, estate_type2){ + public static def processCubagePercent(cubage_percent){ if(cubage_percent){ def v = getValue(cubage_percent) if(v){ - if(estate_type2 == "别墅"){ - if(v>=0.3 && v<=1){ - return v.toString() - } - }else{ - if(v>1 && v<=10){ - return v.toString() - } + if(v>=0.3 && v<=10){ + return v.toString() } } } @@ -418,26 +476,26 @@ class FiledStrand { estateCharege = estateCharege.toDouble() if(estateType2=="普通住宅"){ if(city=="上海" || city=="北京" || city=="广州" || city=="杭州"){ - if(estateCharege>=0.6 && estateCharege<=3.5){ + if(estateCharege>=0.3 && estateCharege<=40){ return estateCharege }else{ return null; } }else{ - if(estateCharege>=0.2 && estateCharege<=3.5){ + if(estateCharege>=0.2 && estateCharege<=20){ return estateCharege }else{ return null; } } }else if(estateType2=="别墅"){ - if(estateCharege>=3){ + if(estateCharege>=1){ return estateCharege; }else{ return null; } }else if(estateType2=="商住"){ - if(estateCharege>=4){ + if(estateCharege>=2){ return estateCharege; }else{ return null; @@ -454,13 +512,13 @@ class FiledStrand { if(household_count){ household_count = household_count.toInteger() if(estateType2=="普通住宅" || estateType2=="商住"){ - if(household_count>=50 && household_count<=100000){ + if(household_count>=15 && household_count<=100000){ return household_count; }else{ return null; } }else if(estateType2=="别墅"){ - if(household_count>=20 && household_count<=100000){ + if(household_count>=10 && household_count<=100000){ return household_count; }else{ return null; @@ -470,6 +528,20 @@ class FiledStrand { return null; } + static def processLoudongCount(loudongCount){ + def v = getValue(loudongCount.toString()); + if(v){ + if(v>=1 && v<=10000){ + try { + return v.toString().toInteger(); + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + static def processFinishBuildingDate(def finishBuildingDate){ if(finishBuildingDate){ Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}"); @@ -493,6 +565,39 @@ class FiledStrand { } + static def processCompany(val){ + if(val){ + if(val.length()>20){ + return null + }else{ + if(val.contains("无")){ + return null + } + def fiterChars = [",",".","?","/","数据",'-','?',' ',' '] + for(def s : fiterChars){ + val = val.replace(s, '') + } + + if(val.length()<1){ + return null + } + + if(val.toString().isNumber()){ + return null + } + + Pattern pattern = Pattern.compile("\\d{4}年"); + if (pattern.matcher(val).matches()) { + return null + } + return val + } + } + + return null; + } + + def converList(def dbCursor){ def list = [] for(def dbo : dbCursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 210802c58..e4e5016ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -143,17 +143,8 @@ class OpenApiUtils { def result = queryMaxSimilarAddress(city, name) if(result){ for(def ret : result){ - if(ret && ret.lat){ - try{ - def gmap = GeocodeUtils.getGaodeToBaidu(ret.lng.toDouble(), ret.lat.toDouble()) - ret.lng2 = gmap.get("lng") - ret.lat2 = gmap.get("lat") - ret.estateType = ret.type - ret.source = "高德" - }catch (Exception e){ - - } - } + ret.estateType = ret.type + ret.source = "高德" } } return result @@ -161,11 +152,17 @@ class OpenApiUtils { public static def queryMaxSimilarAddress(String city, String address, boolean isFullMatch = false){ - if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ +// if(StringUtils.isEmpty(city)){ +// return null +// } + if(StringUtils.isEmpty(address)){ return null; } address = address.trim().replace(" ","") - String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" +// String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" + //key:8796b9dfb3889b57c38e2fff1b7e51bd 花钱买的 + String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&city=" + city + "&output=json&offset=20&page=1&key=8796b9dfb3889b57c38e2fff1b7e51bd&extensions=all&citylimit=true"//citylimit限制城市 + List poiList = null; com.alibaba.fastjson.JSONObject result = null; @@ -553,6 +550,35 @@ class OpenApiUtils { return null; } + static def similarRank(proCode, cityCode, regionCode, name, area, floor, height){ + def url = "http://localhost:8080/fangjia-open/house/similar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" + + "propertyTypeCode=01&province=${proCode}&floorNumber=${floor}&strctureArea=${area}" + println url + String content = new HttpRequest().get(url, "utf-8"); + def dto = [] + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + static def financialRating(city, address){ + def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 52f2ea980..0842f9b06 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-01-01" + static def statDate = "2023-03-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年2月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy index 2817d78ee..73ea03b09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj import com.fangjia.core.Constants import com.fangjia.core.util.DateUtil import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.client.MongoCollection @@ -12,29 +13,106 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") +//def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") +//def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") +//Calendar calendar = Calendar.getInstance() +//calendar.setTime(startDate) +//def data = [[ "公司名称","访问链接", "访问时间", "总量"]] +//def rMap = [:] +//def index = 0 +//OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc -> +// index++ +// def day = doc.c_date.format("yyyy-MM-dd") +// def count = rMap.get(day) +// if (!count) { +// rMap.put(day, 1) +// } else { +// rMap.put(day, count + 1) +// } +// if(index%10000==0){ +// println index+" "+new Date().toString() +// } +//} +// +//rMap.each {k,val-> +// println k+" "+val +//} + +//def startDate = DateUtil.parseDate("20230416", "yyyyMMdd") +//def endDate = DateUtil.parseDate("20230418", "yyyyMMdd") +//Calendar calendar = Calendar.getInstance() +//calendar.setTime(startDate) +//def count = 0 +//def hashSet = [] +//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +//while (startDate.before(endDate)) { +// calendar.add(Calendar.DATE, 1) +// def nextDate = calendar.getTime() +// count++ +// def total = 0 +// OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> +// total++ +// def result = JSONObject.fromObject(doc.res_map.result) +//// if (!hashSet.contains(result)) { +// def item = [doc._id.toString(), "neiping2017@czbank", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] +// data << item +// hashSet << result +// println total+" "+ nextDate +//// } else { +//// println("已存在") +//// } +// } +// if (count % 7 == 0) { +// hashSet.clear() +// } +// +// startDate = nextDate +// +//} +// +////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +//ExcelUtils.write("D:\\浙商调用详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + + + + +def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) -def data = [[ "公司名称","访问链接", "访问时间", "总量"]] -def rMap = [:] -def index = 0 -OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc -> - index++ - def day = doc.c_date.format("yyyy-MM-dd") - def count = rMap.get(day) - if (!count) { - rMap.put(day, 1) - } else { - rMap.put(day, count + 1) - } - if(index%10000==0){ - println index+" "+new Date().toString() - } -} +def count = 0 +def hashSet = [] +def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] -rMap.each {k,val-> - println k+" "+val +OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) + def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item + hashSet << result } +//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + + +//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd") +//def endDate = DateUtil.parseDate("20230506", "yyyyMMdd") +//Calendar calendar = Calendar.getInstance() +//calendar.setTime(startDate) +//def count = 0 +//def hashSet = [] +//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +// +//OPEN_URI_LOG.find(and(eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> +// def result = JSONObject.fromObject(doc.res_map.result) +// def item = [doc._id.toString(), doc.user_id, doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] +// data << item +// hashSet << result +// count++ +// println count +//} +// +// +////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index b28cc2770..269b5be9e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -33,11 +33,14 @@ class AnjukeDistrictMatch { "postgres", "postgres", "org.postgresql.Driver") static main(args) { +// def name = "陆家嘴滨江中心N2座" +// name = name.replaceAll("-?\\w+(号写字楼|号楼|楼|座|栋)","") +// println name def task = new AnjukeDistrictMatch(); // task.run(); // task.runCsv(); // task.updateCredit() -// task.matchCommunity()//查询es小区数据 + task.matchCommunity()//查询es小区数据 // task.run1() // task.updateGreenRateField()//根据第三方数据填补库中字段 // task.updteGreenRateFieldByCsv() @@ -50,7 +53,7 @@ class AnjukeDistrictMatch { // task.updteBuildAreaFieldByCsv() // task.updadteAllAreaFieldByCsv() // task.updatePlotRateFieldByCsv() - task.updateFinishDateFieldByCsv() +// task.updateFinishDateFieldByCsv() } def updateCredit(){ @@ -728,16 +731,17 @@ class AnjukeDistrictMatch { def run(){ - def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv") - def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx" +// def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv") + def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406.xlsx" def index = 0 ExcelUtils.multiThreadingModify(filePath,{row-> index++ if(row['A'] && row['B']){ def city = row['A'] - def name = row['C'] - def region = row['B'] + def name = row['B'] + def region = null +// def region = row['B'] if(name && name.contains("·")){ name = name.replace("·", "") @@ -785,27 +789,27 @@ class AnjukeDistrictMatch { row['AC'] = jsonObject.get("credit") } } - println index +" "+row['Y'] -// if(row['G']){ -// def dbo = CommonDatautils.findDistrictById(row['G']) -// if(dbo){ -// row['H'] = dbo.estate_type2 -// row['I'] = dbo.name_cat -// -// def count = findBigdataCommunity(row['G']) -// if(count){ -// row['E'] = "是" -// } -// } -// } +// println index +" "+row['Y'] + if(row['G']){ + def dbo = CommonDatautils.findDistrictById(row['G']) + if(dbo){ + row['H'] = dbo.estate_type2 + row['I'] = dbo.name_cat + + def count = findBigdataCommunity(row['G']) + if(count){ + row['E'] = "是" + } + } + } } }) } def matchCommunity(){ - def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)3.24_2.xlsx" + def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406-1.xlsx" ExcelUtils.modify(filePath, { row -> - if (row['A'] && row['B'] && row['I']) { + if (row['A'] && row['B'] && !row['O']) { def rets = findAllEsData(row['A'], row['B']) if(rets){ def dbo = rets[0] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy index 231a930c8..55ce0ed8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -480,7 +480,7 @@ class OfficeProcess { } def checkRegion(){ - def citys = ["北京","广州","重庆","天津","成都","南京"]//"上海","深圳", + def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都", for(def city : citys){ def regionPointsMap = findRegionRange(city) def addDatas = findAddDatasByClickhouse(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 6b07a6159..62666390b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -16,7 +16,140 @@ class FieldStat { // task.run() // task.run1() // task.run2() - task.run3() +// task.run3() +// task.offircRun() + task.distRun() + task.shopRun() + } + + def distRun(){ + def citys = CommonDatautils.findAllCity() + def cityMap = [:] + def fileds = ["region","name","alias","address","addr_alias","estate_type2","build_type","block","loop_line_info","street_community","household_count","curr_household_count","virescence_percent","cubage_percent","all_area","build_area","estate_charge","car_site","car_site_count","developer","estate_company","finish_building_date","live_date","start_building_date","open_date","property_life","floor_info","fitment","fitment_materials","build_structure","traffic","facilities","characteristic","sell_address","relate_info","description","total_price","mobile","building_status","licence","build_designer","garden_designer","business_quarter","bay_area","payment_pattern","land_plan_license","prj_plan_license","land_use_license","cons_license","elevators_count","build_date","divisible","status","metro_info","metro_names","cons_company","sells_company","contact_people","parking_fee","delivery_level","fitment_level","land_car_site","underground_car_site","avg_price","sell_price","standard_area","pics","loudong_count","committee","house_structure","usage_card","mainTypeArea"] + for(def c : citys){ + def dbo = findCoverPrecent(c) + if(dbo && dbo.containsField("district_detail")){ + def officeDetail = dbo.district_detail + cityMap.put(c, officeDetail) + }else{ + cityMap.put(c, [:]) + } + } + + def rets = [] + rets << ["城市","数量","区域","名称","别名","地址","地址别名","物业类型","建筑类型","板块","环线位置","街道社区","户数","当前户数","绿化率","容积率","占地面积","建筑面积","物业费","停车位","停车位数","开发商","物业公司","竣工时间","入住时间","开工时间","开盘时间","产权年限","楼层情况","装修情况","装修材料","建筑结构","交通情况","周边配","小区特征","售楼地址","相关信息","描述","总价","电话","建筑状态","预售许可证","建筑单位","园艺设计单位","商圈","户型面积","支付方式","用地规划许可证","工程规划许可证","土地使用证","施工许可证","电梯数量","建造年代","是否可分割","状态","地铁沿线","地铁站名","施工单位","销售企业","联系人","停车费","交付标准","装修标准","地上车位数","地下车位数","均价","出售价格","标准层面积","图片","楼栋总数","居委会","房屋结构","土地使用权证号","主力户型"] + def ret = [] + ret << "city" + ret << "count" + for(def f : fileds){ + ret << f + } + rets << ret + cityMap.each {k,v-> + def row = [] + row << k + row << CommonDatautils.findAllDistrictCount("district", k) + for(def filed : fileds){ + if(v){ + def val = v.get(underlineToCamel(filed)) + if(val){ + row << val.toDouble()*100 + }else{ + row << "" + } + } + } + rets << row + } + + ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "小区") + } + + def shopRun(){ + def citys = CommonDatautils.findAllCity() + def cityMap = [:] + def fileds = ["region","name","alias","address","business_quarter","property_status","rent_type","lease_term","payment_pattern","sell_price","household_count","mobile","floor","floor_info","bay_area","finish_building_date","live_date","divisible","unified_manage","car_site_count","car_site","all_area","build_area","estate_charge","developer","estate_company","metro_info","metro_names","facilities","property_info","traffic","surrounding","business_status","url","description","open_date","sell_address","block","height","loop_line_info","cubage_percent","virescence_percent","characteristic","sells_company","licence","build_designer","cons_company","property_life","insidepos","delivery_level","start_building_date","land_plan_license","prj_plan_license","land_use_license","cons_license","fitment","build_structure","enter_company","build_type","elevators_count","estate_type2","status","standard_area","relate_info","garden_designer","fitment_level","build_date","fitment_materials","rent_price","out_pics","pics","land_car_site","underground_car_site","addr_alias"] + for(def c : citys){ + def dbo = findCoverPrecent(c) + if(dbo && dbo.containsField("shop_detail")){ + def officeDetail = dbo.shop_detail + cityMap.put(c, officeDetail) + }else{ + cityMap.put(c, [:]) + } + } + + def rets = [] + rets << ["城市","数量","区域","名称","别名","地址","商圈","建筑状态","租售类型","租赁年限","付款方式","出售价格","总户数","电话","建筑楼层","楼层状况","开间面积","竣工时间","开业时间","可分割","统一管理","车位数量","车位描述","占地面积","建筑面积","物业费","开发商","物业公司","地铁沿线","地铁站名","配套设施","物业概况","交通情况","周边情况","经营业态","来源url","描述","开盘时间","售楼地址","板块","楼高","环线位置","容积率","绿化率","商业特色","销售企业","销售许可证","设计单位","施工单位","产权年限","交通位置","交付标准","开工时间","用地规划许可证","工程规划许可证","土地使用证","施工许可证","装修","建筑结构","入驻企业","建筑类型","电梯数量","物业类型","状态","标准层面积","相关信息","景观设计单位","装修标准","建造年代","建材装修","出租价格","外景图","其它图","地上车位数","地下车位数","地址别名"] + def ret = [] + ret << "city" + ret << "count" + for(def f : fileds){ + ret << f + } + rets << ret + cityMap.each {k,v-> + def row = [] + row << k + row << CommonDatautils.findAllDistrictCount("shop", k) + for(def filed : fileds){ + if(v){ + def val = v.get(underlineToCamel(filed)) + if(val){ + row << val.toDouble()*100 + }else{ + row << "" + } + } + } + rets << row + } + + ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "商铺") + } + + def offircRun(){ + def citys = CommonDatautils.findAllCity() + def cityMap = [:] + def fileds = ["region","name","alias","latLng","latLng2","address","addrAlias","bedKind","estateType2","buildType","block","businessQuarter","loopLineInfo","licence","sideset","realUsePercent","buildDesigner","consCompany","estateCharge","carSiteCount","carSite","undergroundCarSite","landCarSite","parkingFee","sight","propertyLife","lobbyHeight","standardHeight","clearHeight","standardArea","foreign","register","lightingVentilation","conditionerTime","","elevatorsCount","personElevatorsCount","goodsElevatorCount","openDate","finishBuildingDate","liveDate","allArea","buildArea","cubagePercent","virescencePercent","developer","estateCompany","networkWay","fitmentLevel","buildStructure"] + for(def c : citys){ + def dbo = findCoverPrecent(c) + if(dbo && dbo.containsField("office_detail")){ + def officeDetail = dbo.office_detail + cityMap.put(c, officeDetail) + }else{ + cityMap.put(c, [:]) + } + } + + def rets = [] + rets << ["城市","数量","区域","名称","名称别名","经纬度(百度)","经纬度(高德)","地址","地址别名","写字楼等级","写字楼类型","建筑类别","板块","商圈","环线位置","预售许可证","是否可分割","得房率","设计单位","施工单位","物业费","停车位数","停车描述","地上车位数","地下车位数","车位租金","开间面积","产权年限","大堂层高","标准层高","净高","标准层面积","是否涉外","是否可注册","空调","空调开放时间","空调费用","电梯数量","客梯数","货梯数","开盘时间","竣工时间","入住时间","占地面积","建筑面积","容积率","绿化率","开发商","物业公司","网络","装修状况","结构"] + def ret = [] + ret << "城市" + ret << "数量" + for(def f : fileds){ + ret << f + } + rets << ret + cityMap.each {k,v-> + def row = [] + row << k + row << CommonDatautils.findAllDistrictCount("office", k) + for(def filed : fileds){ + if(v){ + def val = v.get(filed) + if(val){ + row << val.toDouble()*100 + }else{ + row << "" + } + } + } + rets << row + } + + ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "写字楼") + } def run3(){ @@ -136,4 +269,24 @@ class FieldStat { } return obj } + + + public static String underlineToCamel(String param){ + if (param==null||"".equals(param.trim())){ + return ""; + } + int len=param.length(); + StringBuilder sb=new StringBuilder(len); + for (int i = 0; i < len; i++) { + char c=param.charAt(i); + if (c=="_"){ + if (++i + if (d['A']) { + def results = com.fangjia.script.gj.OpenApiUtils.financialRating(d['A'], d['B']+d['C']) + if(results){ + d['D'] = results.consumptionRating + } + } + }) + } + + def similarRunTest(){ + + ExcelUtils.modify("D:\\数据\\江南农商行\\住宅信息V2-差异案例(反馈).xlsx", { d -> + if(d['A']){ + def cityCode = d['U'].toString().toDouble().toInteger() + def proCode = d['V'].toString().toDouble().toInteger() + def regionCode = d['T'].toString().toDouble().toInteger() + def name = d['D'] + def area = d['F'] + def floor = d['H'].toString().toDouble().toInteger() + def height = d['G'].toString().toDouble().toInteger() + def results = com.fangjia.script.gj.OpenApiUtils.similarRank(proCode, cityCode,regionCode, name, area, floor, height) + if(results){ + def str = "" + for(def r : results){ + str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"\n" + } + d['Y'] = str + } + } + }) + } + + def rankRunTest(){ def dists = CommonDatautils.findAllDistrictByCity("上海","district", ["city","region","name"]) //def dists = CommonDatautils.findDistrictByCityAndRegions("上海","district", ["浦东"], ["city","region","name"]) def rets = createCityDatas(dists) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy new file mode 100644 index 000000000..d32f3e459 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -0,0 +1,417 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.bson.types.ObjectId + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 小区字段入库(自动入库) + */ +@BatchOwner(Watcher.GaoJuan) +class DistrictFieldAddBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018) + //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" + + //,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家" + def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源 + + //"alias":"alias", + def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段 + + static main(args) { + runBatch(new DistrictFieldAddBatch(), args) + } + + @Override + void process(Object args) { + def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京" + + + def analysRets = [] + for(def c : citys){ + def totalCount = CommonDatautils.findAllDistrictCount("district", c) +// addFieldFromRonghe(c, totalCount) + analysRets = addFieldFromMatch(c, totalCount, analysRets) + } + println analysRets.toString() + + ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") + } + + def addFieldFromMatch(c, totalCount, analysRets){ + sourceMap.each { seaweedmapSource, esWebSource -> + def esRets = findEsDatas(c, esWebSource) + def matchRets1 = matchApi(esRets) + def matchRets2 = selectRestByCredit(matchRets1) + println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size() + + def updateRets = processFieldsFromMatch(c , matchRets2) + println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%" + def precentMap = fieldUpdate(totalCount, updateRets) + println c +" "+precentMap.toString() + updateDB(c, updateRets) + println "------------------------------------------------------------------------------" + def pre1 = 0 + def pre2 = 0 + if(esRets){ + pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size()) + pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size()) + } + analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2] + } + return analysRets; + } + + + + + def addFieldFromRonghe(c, totalCount){ + sourceMap.each{seaweedmapSource,esWebSource-> + def rhRets = findRongheDists(c, seaweedmapSource) + def esRets = findEsDistrict(c, esWebSource) + def updateRets = processFieldsFromRh(c, rhRets, esRets) + + println "【通过融合关系填充字段】网站:"+seaweedmapSource +" 第三方融合表:"+rhRets.size() +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, rhRets.size()*100/totalCount)+"%" + def precentMap = fieldUpdate(totalCount, updateRets) + println c +" "+precentMap.toString() + updateDB(c, updateRets) + println "----------------------------------------------------------------------------------" + } + } + + def processFieldsFromRh(c, rhRets, esRets){ + def fields = fieldsMap.keySet() as List + def fieldValueMap = [:] + for(def rhDist : rhRets){ + def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields) + if(!dist){ + println c+" "+rhDist.commid+" 不存在" + continue + } + def esDbo = esRets.get(c + "#" + rhDist.id) + if(!esDbo){ + println c+" "+rhDist.id+" 不存在[ES]" + continue + } + def newValues = [:] + fieldsMap.each{seaweedfield,esField-> + def val = dist.get(seaweedfield) + if(!val){ + def esVal = esDbo.get(esField) + if(esVal){ + def newVal = null + if(seaweedfield == "estate_charge"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + newVal = newVal.toString() + } + if(seaweedfield == "build_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(newVal){ + newValues.put(seaweedfield, newVal) + } + } + } + } + if(newValues){ + fieldValueMap.put(dist._id.toString(), newValues) + } + } + + println fieldValueMap.size() + return fieldValueMap; + + } + + def processFieldsFromMatch(c,matchRets){ + def fields = fieldsMap.keySet() as List + fields << "city" + def fieldValueMap = [:] + for(def matchDist : matchRets){ + + def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) + if(!dist){ + println c+" "+matchDist.match_id+" 不存在" + continue + } + + def newValues = [:] + fieldsMap.each{seaweedfield,esField-> + def val = dist.get(seaweedfield) + if(!val){ + def esVal = matchDist.get(esField) + if(esVal){ + def newVal = null + if(seaweedfield == "estate_charge"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + newVal = newVal.toString() + } + if(seaweedfield == "build_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(seaweedfield == "estate_company" || seaweedfield == "developer"){ + newVal = FiledStrand.processCompany(esVal) + } + if(seaweedfield =="virescence_percent"){ + newVal = FiledStrand.processVirescencePercent(esVal) + } + + if(seaweedfield =="cubage_percent"){ + newVal = FiledStrand.processCubagePercent(esVal) + } + + if(seaweedfield =="all_area" || seaweedfield =="build_area"){ + newVal = FiledStrand.processArea(esVal) + newVal = newVal.toString() + } + if(seaweedfield =="household_count"){ + newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2) + } + + if(seaweedfield =="loudong_count"){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(seaweedfield =="car_site_count"){ + newVal = FiledStrand.processCarSite(esVal) + } + if(seaweedfield =="car_site"){ + newVal = FiledStrand.processCarSiteDesc(esVal) + } + + if(seaweedfield =="property_life"){ + newVal = FiledStrand.processPropertyLife(esVal) + } + + if(seaweedfield =="alias"){ + newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias) + } + + if(seaweedfield =="address"){ + newVal = FiledStrand.processAddress(esVal, dist.city) + } + + if(newVal){ + newValues.put(seaweedfield, newVal) + } + } + } + } + if(newValues){ + println dist._id.toString()+" "+newValues.toString() + fieldValueMap.put(dist._id.toString(), newValues) + } + } + + return fieldValueMap; + } + + + + def findRongheDists(def city, def source){ + def keys = new BasicDBObject() + keys.put("commid", 1) + keys.put("city", 1) + keys.put("id", 1) + keys.put("source", 1) + def dbcursor = SEAWEEDMAP_COLL.find(new BasicDBObject().append("city", city).append("source", source), keys) + def rets = [] + for(def d : dbcursor){ + rets << ["commid":d.commid, "city":d.city, "source":d.source, "id":d.id] + } + return rets; + } + + def findEsDistrict(c, esWebSource){ + def map = [:] + ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r-> + if(r.id && !map.containsKey(c+"#"+r.id)){ + map.put(c+"#"+r.id, r) + } + }) + return map + } + + + def findEsDatas(c, esWebSource){ + def list = [] + ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r-> + list << r + }) + return list + } + + def fieldUpdate(total,updateRets){ + + def countMap = [:] + + for(def rm : updateRets){ + def valMap = rm.value + fieldsMap.each{seaweedfield,esField-> + if(valMap.containsKey(seaweedfield)) { + if(countMap.containsKey(seaweedfield)) { + countMap.put(seaweedfield, countMap.get(seaweedfield) + 1) + }else{ + countMap.put(seaweedfield, 1) + } + } + } + } + + println countMap.toString() + def precentMap = [:] + fieldsMap.each() {key,skey-> + if(countMap.containsKey(key)) { + precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) + }else { + precentMap.put(skey, 0) + } + } + return precentMap; + } + + + def matchApi(esRets){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(esRets.size()) + for(def i=0;i=0.9){ + matchRets1 << r + } + } + return matchRets1 + } + + def matching(data){ + def name = data.name + if(data.name && data.name.contains("·")){ + name = data.name.replace("·", "") + } + + def flag = "名称匹配" + def dist = matchDist(data.city, data.region, name) + + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district") + flag = "映射别名匹配" + } + if(!dist) { + def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district") + if(rets && rets.size()==1){ + flag = "别名匹配" + dist = rets[0] + } + } + + if(dist){ + data.flag = flag + data.match_id = dist._id.toString() + data.match_region = dist.region + data.match_name = dist.name + data.credit = 0.95 + }else{ + def jsonObject = OpenApiClient.addressMatch(data.city, name, "property", false,data.region, null) + if(jsonObject!=null && jsonObject.precise==-1){ + if(data.address && isStandardRoad(data.address)){ + jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false,data.region, name) + }else{ + data.flag = "匹配接口-地址" + } + }else{ + data.flag = "匹配接口-名称" + } + if(jsonObject != null && jsonObject.precise!=-1) { + data.match_id = jsonObject.get("id") + data.match_region = jsonObject.get("district") + data.match_name = jsonObject.get("name") + data.credit = jsonObject.get("credit") + } + } + if(data.match_id){ + return data + } + return null + } + + + def updateDB(c, updateRets){ + + for(def rm : updateRets) { + def id = rm.key + def valMap = rm.value + + BasicDBObject udata = new BasicDBObject() + valMap.each {k,v-> + udata.put(k,v) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata)) + } + } + + def matchDist(city, region, name){ + def dist = CommonDatautils.findDistrict(city, region, name, "district") + return dist; + } + + + + def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy new file mode 100644 index 000000000..945227e7a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.gj.district + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class SameDistName { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + + static main(args) { + def task = new SameDistName() + task.run() + } + + def run(){ + def dists = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region", "name", "address","lat", "lng"]) + def map = [:] + for(def d : dists){ + def list = map.get(d.name) + if(!list){ + list = [] + map.put(d.name, list) + } + list << d + } + + def datas = [] + map.each {k,v-> + if(v && v.size()>=2){ + datas.addAll(v) + } + } + + def retDats = [] + retDats << ["城市","区域","名称","地址","区域状况"] + if(datas){ + def regionPointsMap = findRegionRange("常州") + for(def d : datas){ + def lng = d.lng/10000000000.0 + def lat = d.lat/10000000000.0 + def flag = isInArea(lng, lat, regionPointsMap.get(d.region)) + retDats << [d.city, d.region, d.name, d.address, flag] + } + } + ExcelUtils.write("D:\\数据\\小区\\相同小区不同区域.xlsx", retDats, "数据") + } + + def findRegionRange(city){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion(city,d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy index c133d7283..aa3c3c62c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.gj.es +import com.fangjia.core.util.DateUtil import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xpack.sql.jdbc.EsDataSource @@ -8,16 +10,39 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; -findLogs() +//findLogs() +//findFangjiaLogs() +stat() +def stat(){ + def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1) + def map=[:] + for(def d : datas){ + def count = map.get(d['C']) + if(!count){ + map.put(d['C'], 1) + }else{ + map.put(d['C'], count+1) + } + } + + map.each {k,v-> + println k +" "+v + } +} def findLogs(){ def total = 0 try{ // String sql = "select * from fangjia-log* where request like '/jnBank%'" // BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + +// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd") +// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd") +// QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false); + def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com") - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); println houses.size() def set = new HashSet() @@ -39,4 +64,48 @@ def findLogs(){ } return total +} + + +def findFangjiaLogs(){ + + def startDate = DateUtil.parseDate("20230329", "yyyyMMdd") + def endDate = DateUtil.parseDate("20230429", "yyyyMMdd") + Calendar calendar = Calendar.getInstance() + calendar.setTime(startDate) + def list = [] + def data = [["mid", "ip", "访问时间"]] + while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + +// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd") +// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd") + QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false); + + + def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); + println houses.size() + + if (houses) { + for (def d : houses) { + def reqStr = d.request.toString() + def mid = null + if (reqStr.contains("mid=")) { + + mid = reqStr.substring(reqStr.indexOf("mid=") + 4, reqStr.indexOf("mid=") + 8) + } + def time = d.get("@timestamp").toString() + time = time.substring(0, time.indexOf("T")) + def ip = d.remote_addr + data << [mid, ip, time] + } + } + + startDate = nextDate + } + + ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志") } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy index 444fddbfc..ea7d46e2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.es +import com.fangjia.core.util.DateUtil import com.fangjia.script.util.ElasticUtils import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder @@ -9,14 +10,18 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; -findLogs() - +//findLogs() +findFangjiaLogs() def findLogs(){ def total = 0 try{ // String sql = "select * from fangjia-log* where request like '/jnBank%'" // BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + def startDate = DateUtil.parseDate("20230427", "yyyyMMdd") + def endDate = DateUtil.parseDate("20230428", "yyyyMMdd") + + QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false); def q = QueryBuilders.fuzzyQuery("request", "/jnBank") QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder); @@ -53,3 +58,35 @@ def findLogs(){ return total } + + + + + +def findFangjiaLogs(){ + def total = 0 + try{ + +// String sql = "select * from fangjia-log* where request like '/jnBank%'" +// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + def startDate = DateUtil.parseDate("20230427", "yyyyMMdd") + def endDate = DateUtil.parseDate("20230428", "yyyyMMdd") + + QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false); + def q = QueryBuilders.fuzzyQuery("request", "/enquire") + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder); + println houses.size() + def set = new HashSet() + if(houses) { + for (def d : houses) { + + } + } + println set + }catch(Exception e){ + e.printStackTrace() + } + + return total +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy new file mode 100644 index 000000000..ee95b0621 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy @@ -0,0 +1,556 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject + +import java.util.regex.Pattern + +class GaoDeMapSearchTest { + + def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] + + + static main(args) { + def task = new GaoDeMapSearchTest(); + def path = "D:\\数据\\光大\\光大测试-V2.xlsx" + +// task.structureV2(path) +// task.match(path) +// task.mapSearch(path) + task.baiduMapSearch(path) + task.checkTrue(path) + } + + def checkTrue(path){ + ExcelUtils.multiThreadingModify(path,{row-> + if(!row['Z']){ + if(row['Q']){ + row['Z'] = "T" + }else{ + if(row['Y']){ + def address = '' + def poi = row['G'] + if(row['E']){ + address = address + row['E'] + } + if(row['F']){ + address = address + row['F'] + } + if(address && isStandardRoad(address)){ + def mapAddr = row['U'] + if(mapAddr && mapAddr.contains(address)){ + row['Z'] = "T" + } + }else if(poi){ + def mapName = row['T'] + if(mapName && mapName==poi){ + row['Z'] = "T" + } + } + } + } + } + }) + } + + + def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + + + def mapSearch(filePath){ + ExcelUtils.multiThreadingModify(filePath,{row-> + if(!row['M'] && !row['R']){ + def city = row['C'] + def region = row['D'] + def address = '' + def poi = row['G'] + if(row['E']){ + address = address + row['E'] + } + if(row['F']){ + address = address + row['F'] + } + def rets = [] + if(address){ + def ret1 = OpenApiUtils.getGaoDeSearchLocation(city, region, address) + rets.addAll(ret1) + } + if(!rets && poi){ + def ret2 = OpenApiUtils.getGaoDeSearchLocation(city, region, poi) + rets.addAll(ret2) + } + +// if(!map && rets && rets.size()>0){ +// map = rets[0] +// } + + def bdRes = searchMapBySimilar(city, region, poi, address, rets); + if (bdRes) { + row['R'] = bdRes.uid + row['S'] = bdRes.region + row['T'] = bdRes.name + row['U'] = bdRes.address + row['V'] = bdRes.type + row['W'] = bdRes.score + if(bdRes.matchType){ + row['X'] = bdRes.matchType + } + row['Y'] = "高德" + } + } + },0) + } + + + + def findStandardObject(def city, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + //相似里找有精确地址 + for (def mapObj : rets) { + mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address); + if (mapObj.address && findPattern(mapObj.address)) { + map = mapObj; + break; + } + } + if(!map){ + //交带有交叉口地址 + for (def mapObj : rets) { + if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) { + if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){ + map = mapObj; + break; + } + } + } + } + } +// if (!map) { +// map = rets[0] +// } + } + return map; + } + + + def structureV2(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def jsonObj = FangjiaClientUtils.addressStructure(null, row['A']) + if(jsonObj && jsonObj.result){ + if(jsonObj.result.province){ + row['B'] = jsonObj.result.province + } + if(jsonObj.result.city){ + def cityStr = null + if(jsonObj.result.city=="市辖区"){ + cityStr = jsonObj.result.province + }else{ + cityStr = jsonObj.result.city + } + row['C'] = RegionData.formatCity(cityStr) + } + + if(jsonObj.result.district){ + row['D'] = jsonObj.result.district + } + + if(jsonObj.result.street){ + row['E'] = jsonObj.result.street + } + + if(jsonObj.result.streetNumber){ + row['F'] = jsonObj.result.streetNumber + } + + if(jsonObj.result.name){ + row['G'] = jsonObj.result.name + } + + if(jsonObj.result.buildingNumber){ + row['H'] = jsonObj.result.buildingNumber + } + + if(jsonObj.result.unitNumber){ + row['I'] = jsonObj.result.unitNumber + } + if(jsonObj.result.roomNumber){ + row['J'] = jsonObj.result.roomNumber + } + + if(jsonObj.result.formatAddress){ + row['K'] = jsonObj.result.formatAddress + } + + if(jsonObj.result.addrVerify){ + row['L'] = jsonObj.result.addrVerify + } + } + } + }) + } + + + + //可以根据测试数据添加 推荐楼盘名 + def match(path){ + ExcelUtils.modify(path, { row -> + def city = row['C'] + if(city && !row['M']){ + def address = '' + def suggestName = row['G'] + if(row['E']){ + address = address + row['E'] + } + if(row['F']){ + address = address + row['F'] + } + def region = row['D'] + if(!row['M'] && address){ + city = RegionData.formatCity(city) + if(city){ + region = RegionData.formatRegion(city, region) + } + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, suggestName)//"property" + Thread.sleep(2000) + if(jsonObject != null && jsonObject.precise!=-1) { + row['M'] = jsonObject.get("city") + row['N'] = jsonObject.get("district") + row['O'] = jsonObject.get("name") + row['P'] = jsonObject.get("propertyType") + row['Q'] = jsonObject.get("id") + } + } + } + }) + } + + def baiduMapSearch(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C'] && !row['M'] && !row['R']){ + def city = row['C'] + def region = row['D'] + def address = '' + def poi = row['G'] + if(row['E']){ + address = address + row['E'] + } + if(row['F']){ + address = address + row['F'] + } + def baiduCode = getBaiduMapCode(city); + if(baiduCode){ + def rets = null + if(address && row['G']){ + rets = getBiduSearchDatas(baiduCode , address, row['G'], null) + }else if(row['G']){ + rets = getBiduSearchDatas(baiduCode , null, row['G'], rets) + }else if(address){ + rets = getBiduSearchDatas(baiduCode , null, address, rets) + } + + def bdRes = searchMapBySimilar(city, region, poi, address, rets); + if (bdRes) { + row['R'] = bdRes.uid + row['S'] = bdRes.region + row['T'] = bdRes.name + row['U'] = bdRes.address + row['V'] = bdRes.type + row['W'] = bdRes.score + if(bdRes.matchType){ + row['X'] = bdRes.matchType + } + row['Y'] = "百度" + } + println row['B'] +" "+ bdRes + } + } + }); + } + + + def getSimpleAddress(city, address){ + def jsonObj = FangjiaClientUtils.addressStructure(city, address) + def old_addr = "" + def map = [:] + if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + map.pro = jsonObj.result.province + } + + if(jsonObj.result.city){ + if(jsonObj.result.city=="市辖区"){ + map.city = jsonObj.result.province + }else{ + map.city = jsonObj.result.city + } + map.city = RegionData.formatCity(map.city) + } + + if(jsonObj.result.district){ + map.region = jsonObj.result.district + } + if(jsonObj.result.street){ + def street = jsonObj.result.street + if(street){ + old_addr = street + map.street = street.replace("交叉口","").replace("交叉","") + } + } + if(jsonObj.result.streetNumber){ + def street_number = jsonObj.result.streetNumber + if(street_number){ + old_addr = old_addr + street_number + } + } + } + map.addr = old_addr.replace("交叉口","").replace("交叉","") + return map; + } + + def getBaiduMapCode(city){ + if(city){ + return baiduCityCodeMap[city] + } + return city + } + + def getBiduSearchDatas(baiduCode , simpleAddr, nameKey, rets){ + def searchKey = "" + if(simpleAddr){ + searchKey = simpleAddr +" "+nameKey + }else{ + searchKey = nameKey + } + def datas = searchBaiDuMap(searchKey, baiduCode.toInteger()); + if(!rets){ + return datas; + }else{ + def ids = [] + for(def r : rets){ + if(!ids.contains(r.uid)){ + ids << r.uid + } + } + for(def d : datas){ + if(!ids.contains(d.uid)){ + rets << d + } + } + return rets; + } + return null; + } + + + static def searchBaiDuMap(String address, int cityCode) { + def rets = [] + try { + Thread.sleep(500) + // String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&&auth=Dae1RaNR1LK8xdJTxN8SQOTRa2AH%3DL7vuxLNVENTxRBt6BG6B6WJWwGvzBU%40yYxvkGcuVtvvhguVtvyheuVtcvY1SGpuLtB2QKAZzvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwBVDzvFosSE21wiKDv7uvhgMuzVVtvrMhuENLtSrrlrIKHf0wd0vyMO7FOUySy&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a"; + String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=19&from=webmap&sug_forward=&auth=E8LCawOwRbEOx6JVTvCIBI6W7zSz5vSXuxLNVENTRVLtUDHUDUYKYyHxzBU%40yYxvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwDWBWvveh3uxtwiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3VP8JSrrlrIKHf0wd0vyMO7FOUS7F&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a&device_ratio=2&tn=B_NORMAL_MAP&nn=0&u_loc=13513133,3631861&ie=utf-8&t=${System.currentTimeMillis()}&newfrom=zhuzhan_webmap" + def content = new HttpRequest().get(url, null, "utf-8") + JSONObject json = JSONObject.fromObject(content) + if (json && json.content && json.content.size() > 0) { + for (def loc : json.content) { +// String name = formatName(address) + def x = loc.x + def y = loc.y + if(x){ + x = x/100 + } + if(y){ + y = y/100 + } + if(x && y){ + def locJson = BaiDuApiUtils.geoconv(x, y, 6) + if(locJson){ + def baiduCity = RegionData.parseCityInAddress(loc.area_name) + def bean = [:] +// def lng2 = locJson.get("x") +// def lat2 = locJson.get("y") +// def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2) +// def lng = gmap.get("lng") +// def lat = gmap.get("lat") + bean.uid = loc.uid +// bean.baidu_lng2 = lng2 +// bean.baidu_lat2 = lat2 +// bean.longitude = lng +// bean.latitude = lat + bean.name = loc.name +// if(loc.address_norm){ +// bean.address = getAddressInfo(loc.address_norm, loc.addr); +// } + bean.address = loc.addr + + if(!baiduCity){ + baiduCity = RegionData.formatCity(loc.city_name) + } + bean.cityname = baiduCity + if(loc.city_name){ + def regionStr = loc.area_name.replace(loc.city_name, "") + def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); + bean.region = baiduRegion + } + + if(loc.std_tag){ + bean.type = loc.std_tag + } + bean.flag = loc.acc_flag + rets << bean; + } + } + } + } + } catch(Exception e) { + searchBaiDuMap(address, cityCode) + } + return rets; + } + + + def searchMapBySimilar(def city, def region, def newKey, def inpAddr, def rets){ + def bo = null; + if(rets){ + def maxScore = 0; + if(!bo || !maxScore){ + for(def o :rets){ + def addr = o.address + def mapKey = o.name + if(mapKey){ + def score = 0 + def nameScore = 0 + if(newKey){ + nameScore = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey); + } + + def addr_score = 0 + if(inpAddr) { + def baidu_addr_map = getSimpleAddress(o.cityname, o.address) + if(!baidu_addr_map.addr){ + baidu_addr_map.addr = AddressUtil.formatAddress(o.cityname, o.address) + } +// def baidu_addr_map = AddressUtil.formatAddress(o.cityname, o.address) + if(!baidu_addr_map.addr){ + if(o.region && !region.contains(o.region)){ + continue; + } + if(city!=o.cityname){ + continue; + } + } + if (baidu_addr_map.addr) { + if(baidu_addr_map.addr.contains("与")){ + def arrs = baidu_addr_map.addr.split("与") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, s1); + if(temp_addr_score > addr_score && (inpAddr.contains(baidu_addr_map.addr) || baidu_addr_map.addr.contains(inpAddr))){ + addr_score = temp_addr_score + } + } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr_map.addr); + if(addr_score>0 && inpAddr && baidu_addr_map.street && !inpAddr.contains(baidu_addr_map.street) && !baidu_addr_map.street.contains(inpAddr)){ + addr_score = 0 + } + } + + } + } + + if(addr_score>0 && score==0){ + addr_score = addr_score/2 + } + def total_score = nameScore + addr_score +// if(city==o.cityname){ + if(total_score > maxScore){ + maxScore = total_score; + o.city = o.cityname + bo = o + bo.score = maxScore + + def list = [] + if(addr_score>0){ + list << "地址有关" + } + if(nameScore>0){ + list << "名称相似" + } + if(list){ + o.matchType = list.join(",") + } + } +// if(bo && score!=1){ +// if(inpAddr){ +// def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) +// if(baidu_addr){ +// def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); +// //if(addr_score==1){ +// // o.matchType = "城市相同,名称相似,地址有关" +// bo.score = bo.score + addr_score +// maxScore = bo.score; +// //} +// } +// } + +// for(def word : filters){ +// if(mapKeyName.toString().endsWith(word)){ +// mapKeyName = mapKeyName.replace(word, "") +// } +// if(newName.toString().endsWith(word)){ +// newName = newName.replace(word, ""); +// } +// } +// +// +// if(mapKeyName && newName && mapKeyName.length()>=2){ +// score = CosineSimilarAlgorithm.getSimilarityRatio(mapKeyName, newName); +// if(score > maxScore){ +// maxScore = score +// o.matchType = "名称相似" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// }else{ +// if(mapKeyName.length() == newName.length()){ +// def mapKeyNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(mapKeyName, " ") +// def newNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") +// if(mapKeyNameBY == newNameBY){ +// maxScore = 0.977; +// o.matchType = "拼音相似" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// } +// } +// } +// } +// } +// } + } + } + } + } + return bo; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index f641c214c..2aae0edb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -32,7 +32,7 @@ public class BeiKeCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") static main(args) { def task = new BeiKeCrawlByChrome(); @@ -44,7 +44,7 @@ public class BeiKeCrawlByChrome { def run(){ - System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() @@ -112,18 +112,20 @@ public class BeiKeCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) for(def m : citys){ - driver.get(m.url+"/chengjiao/") + def city = m[0] + def url = m[1] + driver.get(url+"/chengjiao/") Thread.sleep(10000) def doc = Jsoup.parse(driver.getPageSource()); def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ def regionText = r.text() - def href = m.url+ r.attr("href") + def href = url+ r.attr("href") - println "开始抓取【${m.city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, m.city, href, regionText) - println m.url +" 详情链接:"+detailUrls.size() + println "开始抓取【${city}】【${regionText}】【${href}】" + def detailUrls = getPageDeail(driver, city, href, regionText) + println url +" 详情链接:"+detailUrls.size() } } @@ -183,7 +185,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, region) @@ -398,16 +400,13 @@ public class BeiKeCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) def list = [] + def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"] if(cityEles){ for(def e : cityEles){ - def href = e.attr("href") -// if(e.text().contains("福州")){ - def cityMap = [:] - cityMap.city = e.text() - cityMap.url = "https:"+href - list << cityMap -// cityMap.put(e.text(), "https:"+href) -// } + def href = "https:"+e.attr("href") + if (citys.contains(e.text())){ + list << [e.text(), href] + } } } return list @@ -432,6 +431,7 @@ public class BeiKeCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; + Thread.sleep(10000) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 3c869e61b..343a4a3a4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -31,7 +31,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -117,10 +117,11 @@ public class LianjiaCrawlByChrome { } def run(){ - System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) +// cityMap = cityMap.reverse() getLogin(driver) crawlData(driver, cityMap) @@ -184,12 +185,15 @@ public class LianjiaCrawlByChrome { } } - def crawlData(driver, cityMap){ + def crawlData(driver, cityList){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - cityMap.each{city, cityUrl-> + for(def ret : cityList){ + def city = ret[0] + def cityUrl = ret[1] //def detailUrls = getPageUrls(driver, cityUrl) driver.get(cityUrl+"chengjiao/") + Thread.sleep(10000) def doc = Jsoup.parse(driver.getPageSource()); def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ @@ -251,7 +255,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) @@ -473,17 +477,17 @@ public class LianjiaCrawlByChrome { Thread.sleep(8000) def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) - def cityMap = [:] - def citys= ["苏州"] + def cityList = [] + def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") if (!href.contains("fang.lianjia") && citys.contains(e.text())){ - cityMap.put(e.text(), href) + cityList << [e.text(), href] } } } - return cityMap + return cityList } def getLogin(driver){ @@ -505,6 +509,7 @@ public class LianjiaCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; + Thread.sleep(20000) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index b994fde09..f419acf6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -21,10 +21,10 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\招商\\招商.xlsx" -// addGXDPrice(filePath) + def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx" + addGXDPrice(filePath) // addCcbPrice(filePath) - addThirdPrice(filePath) +// addThirdPrice(filePath) } @@ -104,12 +104,13 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['M']) { - def dist = CommonDatautils.findDistrictById(d['M']) - if(dist){ - def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) - d['AO'] = gxdPrice - } + if (d['B']) { +// def dist = CommonDatautils.findDistrictById(d['B']) +// if(dist){ +// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) + def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) + d['G'] = gxdPrice +// } } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy index 1bdbfcdc7..07ab82cce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy @@ -23,11 +23,16 @@ class ChongQingBlankTest { static main(args) { def task = new ChongQingBlankTest() task.run() + } def run(){ - ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row-> + allRegionMap.each{k,v-> + println k+" "+v.size() + } + + /* ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row-> if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){ def city = CommonUtils.parseCityInAddress(row["B"]) if(!city){ @@ -197,7 +202,7 @@ class ChongQingBlankTest { } } } - }, 0) + }, 0)*/ } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a254cbdd7..175b2ebd1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -27,8 +27,7 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] - def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] - + def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"] static main(args) { runBatch(new SynchDistrictPrice(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 5f9a10997..bdd887539 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202301); - def count2 = findTotalCount(source, 202302); + def count1 = findTotalCount(source, 202302); + def count2 = findTotalCount(source, 202303); println source +" "+count1 +" "+count2 } }