diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy index 6dcae7f53..a11b60d8d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy @@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ServerAreaMap import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * Created by dxy on 2016/12/15. */ @@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch { private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def DISTRICT_STAT_MAP = [:] public void process(Object args) { - def cities = ServerAreaMap.findAllCities() + + + def cities = ["深圳"]//ServerAreaMap.findAllCities() for(String city : cities) { - DISTRICT_STAT_MAP.clear() - Map> regions = RegionData.getAllRegions(city) - regions.each {region, alias-> - processRegionRank(city, region) - } - processCityRank(city) +// DISTRICT_STAT_MAP.clear() +// Map> regions = RegionData.getAllRegions(city) +// regions.each {region, alias-> +// processRegionRank(city, region) +// } +// processCityRank(city) + + processCityRentRank(city) } } @@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch { return ret } + def processCityRentRank(city){ + def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd") + def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth) + def prices = [] + distRentsMap.each{key, price-> + if(!prices.contains(price)){ + prices << price + } + } +// prices = prices.unique() + prices = prices.sort().reverse() + + def totalCount = new AtomicLong() + def keys = distRentsMap.keySet().asList() + final CountDownLatch latch = new CountDownLatch(keys.size()) + +// for(def i=0;i + index ++ + def arrs = key.split("##"); + double percentile = calculatePercentile(prices, price); + + DBObject q = new BasicDBObject("city", arrs[0]) + q.append("region", arrs[1]) + q.append("district_name", arrs[2]) + + DBObject udata = new BasicDBObject() + udata.put("rent_city_rank", (int)percentile); + udata.put("m_date", new Date()) + DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false) + println index +" "+ arrs[2] + } + } + + def upateDistrict(key, distRentsMap, prices){ + def arrs = key.split("##"); + def price = distRentsMap.get(key) + + double percentile = calculatePercentile(prices, price); + + DBObject q = new BasicDBObject("city", arrs[0]) + q.append("region", arrs[1]) + q.append("district_name", arrs[2]) + + DBObject udata = new BasicDBObject() + udata.put("rent_city_rank", (int)percentile); + udata.put("m_date", new Date()) +// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false) + } + + public static int calculatePercentile(def values, def targetValue) { + // 找出数字在List中的排名 + def rank = values.indexOf(targetValue) + 1 + // 计算百分位数 + def percentile = (int)((rank / values.size()) * 100) + return percentile; + } static main(args) { runBatch(new DistrictStatRankBatch(), args) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c52b21fcb..17860e977 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) -// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 - def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 +// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt 生成->导Oracle" - delPgData() - saveToPgSql(rows) +// delPgData() +// saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 6854ae591..4d289d469 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -66,50 +66,51 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } println "curDate:"+curDate + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } + } +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 // for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// // } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - for(def city : citys){ - index ++ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成 " + rows.size() - - saveToPgSql(rows) +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +// saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) - def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) + def trades = CcbCommonUtils.findTradesByMonth(city, startDate) final CountDownLatch latch = new CountDownLatch(trades.size()) def final AtomicInteger count_ = new AtomicInteger(0) @@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{ rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address); } } - if(rMap){ + if(rMap && rMap.region==region){ CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index cdad2a607..1e818b3ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -28,9 +28,11 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); - task.deleteThirdPartyData() +// task.deleteThirdPartyData()//删除第三方 // task.delDealByPe() // task.delDealData() + + task.exportThirdParty() } def delDealByPe(){ @@ -372,15 +374,29 @@ class DealPriceDataProcess { return map } + def exportThirdParty(){ + def citys = CcbCommonUtils.citys_all + def results = [] + results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE'] + for(def city : citys) { + def rets = findPeGteThirdParty3(city, 202305) + for(def map : rets){ + results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe] + } + println city +" "+ rets.size() + } + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方") + } def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202303 + def month = 202304 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202304) + def rets = findPeGteThirdParty2(city, 202305) def total = 0 println city +" "+rets.size() if(rets){ @@ -431,6 +447,28 @@ class DealPriceDataProcess { return rets } + //城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE + + def findPeGteThirdParty3(cityName, yearmonth){ + def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15" + def rets = [] + pgsqlInstance.eachRow(sql,{r-> + def map = [:] + map.cityid = r.urbn_id + map.city = r.urbn_nm + map.data_id = r.gujia_fsid + map.region = r.gujia_rgon_nm + map.comm_id = r.gujia_comm_id + map.district_name = r.gujia_comm_nm + map.type = r.gujia_bds_clc_tp + map.clc_price = r.gujia_bds_avprc + map.price = r.gujiabds_to_mkprc_mny + map.ppe = r.gujia_mns_rto + rets << map + }); + return rets + } + def delDealData(){ ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 4df84ea86..8c765a987 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -79,9 +79,9 @@ class ExportOracleAllDataTest { // exportCommCaseDetailGj()// // // ok文件 - exportInOk() - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// exportInOk() +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 5c2e8c26c..60e475036 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -33,7 +33,7 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() -// task.addFjwPrice() + task.addFjwPrice() // task.addCcbPrice()//oracle // task.addCcbPriceFromGuided() @@ -42,7 +42,7 @@ class FjwPricePkCcbPrice { // task.exportFjwPriceAndCcbPrice() // task.addNewDistPrice() - task.addCcbPrice2()//按id查 +// task.addCcbPrice2()//按id查 // task.addCcbPrice3()//按city,region,name查 // task.standRegion() // task.addGuidPrice() @@ -217,17 +217,19 @@ class FjwPricePkCcbPrice { } def addFjwPrice(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d-> - if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['C']) + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d-> + if(!d['N']){ + def dist = CommonDatautils.findDistrictById(d['E']) if(dist){ - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd")) - if(dbo){ - d['K'] = dbo.avg_price - } + d['N'] = dist.region +// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd")) +// if(dbo){ +// d['K'] = dbo.avg_price +// } + + } } - }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 953a53a6b..91fb02cdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -83,9 +83,9 @@ class GuiDingPriceBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); saveToPgSql(rows) - - //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") +// +// //更新静态数据表 +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 852f3783a..ae9f4f058 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy new file mode 100644 index 000000000..a009eb954 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy @@ -0,0 +1,261 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql + +class FumingTest { + + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + + static main(args) { + def task = new FumingTest(); + task.run(); + } + + def run() { + def path = "D:\\数据\\富民\\富民测试500-v7.xlsx" +// match(path) +// addPrice(path) +// addThirdPrice(path) +// addLastPrice(path) + addFacLevel(path) + } + + + def addFacLevel(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['K'] && row['I']){ +// def obj = findCommunitySummary(row['C']) +// if(obj){ +// row['K'] = obj.consumption_rating +// }else{ + row['K'] = getNearByDistrictLevel(row['C'], row['I']) +// } + } + }) + } + + + /** + * 一公里小区数量 + */ + def getNearByDistrictLevel(def id, def price){ + BasicDBObject district = DbUtils.seaweed.findById(id); + def level = null + if(district){ + List list = DbUtils.findAroundDistrict(district); + println id +" 共:"+list.size()+" "+ price + if(list!=null&&list.size()>0){ + for(def obj : list){ + def summary = findCommunitySummary(obj._id.toString()) + if(summary && summary.avg_price && summary.consumption_rating){ + def pre = java.lang.Math.abs((summary.avg_price - price)*100/price) + println summary.name +" "+pre +" "+ summary.avg_price + if(pre <= 10){ + level = summary.consumption_rating + break; + } + } + } + } + } + return level; + } + + + def addLastPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Z'] ){ + if(row['W']){ + row['Z'] = row['W'] + }else if(row['R']){ + row['Z'] = row['R'] + }else if(row['I']){ + row['Z'] = row['I'] + } + + if(row['Z']){ + row['J'] = (int)(row['Z']*row['B'].toString().toDouble()) + } + } + }) + } + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['N'] ){ + def city = row['C'] + def address = row['A'] + def region = row['D'] + def name = row['G'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) + if(jsonObject != null && jsonObject.precise!=-1) { + row['N'] = jsonObject.get("id") + row['O'] = jsonObject.get("city") + row['P'] = jsonObject.get("district") + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("propertyType") + row['S'] = jsonObject.get("address") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['C'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + row['D'] = dist.city + row['E'] = dist.region + row['F'] = dist.name + row['G'] = dist.estate_type2 + row['H'] = dist.address +// row['Q'] = dist.name + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['I'] = priceObj.avg_price + row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble()) + row['R'] = priceObj.avg_price_2b + } + } + } + }) + } + + + def addThirdPrice(path){ +// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连'] + def allMap = [:] +// for(def c : citys){ +// def cityMap = findThirdPartyPrice(c) +// allMap << cityMap +// } + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F']) + def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['L'] = priceMap.get("lianjia_esf_price") + } + + if(priceMap.containsKey("beike_esf_price")){ + row['M'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("anjuke_esf_price")){ + row['N'] = priceMap.get("anjuke_esf_price") + } + } + def map = findAvgPrice(row['C'], cityCodeMap.get(row['D'])) + if(map){ + row['O'] = map.avg_price + row['P'] = map.credit_value + } + }) + } + + + + def addCcbPrice2(){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d -> + if(d['C']){ + def map = findAvgPrice(d['C'], cityCodeMap.get(d['E'])) + if(map){ + d['K'] = map.avg_price + d['L'] = map.calc_type +// d['AI'] = map.credit_value + d['M'] = map.month + } + } + }) + } + + def findThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + } + return map + } + + def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + + + def findCommunitySummary(id){ + def obj = null + def sql = "select * from community_summary where id='${id}'" + ElasticUtils.executeSqlQuery(sql, { + obj = it + }) + return obj + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy index 112de372f..50db279aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -26,7 +26,7 @@ class MingShenPocTest { def run(){ def path = "D:\\数据\\民生\\poc测试.xlsx" //9级标准化 - //structureV2(path)//针对海南poc地址数据此接口不合适使用 + structureV2(path)//针对海南poc地址数据此接口不合适使用 //省市区拆分 // ner(path)//城市,区域带市带区 // match(path) @@ -34,7 +34,7 @@ class MingShenPocTest { // addLatLng(path) // addAroundPrice(path) // addAuthor(path) - addHouse(path) +// addHouse(path) } def structureV2(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 543973cfa..6c633d4f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -73,6 +73,8 @@ class CommonDatautils { public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo); + public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -230,6 +232,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); + q.put("name", "长滩美墅三期") DBObject keys = new BasicDBObject(); if(fields){ @@ -781,7 +784,7 @@ class CommonDatautils { public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); q.put("region", new BasicDBObject('$in', regions)) @@ -3418,11 +3421,27 @@ class CommonDatautils { dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def sqmetreRentMap = [:] for(def d : dbcursor){ - sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) +// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) + sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price) } return sqmetreRentMap; } + + static def findAllSqmetreRentPrices(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("cat","district") + q.put("weekend", weekend) + def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000) + dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def prices = [] + for(def d : dbcursor){ + prices << d.avg_price + } + return prices; + } + static def findNewSqmetreRent(def city, def region, def name){ def q = new BasicDBObject() q.put("city", city) @@ -3490,5 +3509,14 @@ class CommonDatautils { } return obj } + + static def findDistrictAnalyStat(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q) + return analyData + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index ea6d2717b..464ac2301 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -82,20 +82,21 @@ class DistrictInfoExportBatch extends BaseBatch{ } } def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - for(def dist : removeDists){ - def key = dist.city +"##"+dist.region+"##"+dist.name - if(!distKeys.contains(key)){ - distKeys << key - districts << dist - } - } +// for(def dist : removeDists){ +// def key = dist.city +"##"+dist.region+"##"+dist.name +// if(!distKeys.contains(key)){ +// distKeys << key +// districts << dist +// } +// } def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 - saveOrUpdateFzDist(districts) +// saveOrUpdateFzDist(districts) + remarkRemoveDist(removeDists) def fzDistricts = getFzDistricts(city) for(def rMap : fzDistricts){ @@ -185,6 +186,15 @@ class DistrictInfoExportBatch extends BaseBatch{ } + def remarkRemoveDist(removeDists){ + for(def rd : removeDists){ + def udate = new BasicDBObject() + udate.put("status",1) + udate.put("m_date", new Date()) + FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate)); + } + } + def getFzDistricts(def city){ def q = new BasicDBObject() q.put("city", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 96ed55fcb..dc389ffe5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch { if(weekendPriceObj.value_type!=1){ if(dist.city == "福州"){ if(Math.abs(mom)*100>3){ - def randMarkup = nextDouble(0, 3) - if(mom < 0){ - mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) - }else{ - mom = DataUtil.roundDouble(randMarkup/100, 4) + if(dist.region != "马尾"){ + def randMarkup = nextDouble(0, 3) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100, 4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) } - curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) } }else{ if(Math.abs(mom)*100>10){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 917ddfc5e..2da506d11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); - task.regionPriceTrendRun() +// task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 47a3b331f..9cea6cf74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.core.RegionData import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils @@ -11,26 +12,91 @@ import com.mongodb.DBCollection import org.apache.commons.io.FileUtils import org.bson.types.ObjectId +DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") -DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") -def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") -def rets = [] -for(def d : datas){ - def arrs = d.split("@") - rets << ["commId":arrs[0], "avg_price":arrs[4]] -} - -for(def r : rets){ - if(r.avg_price){ - def udata = new BasicDBObject() - udata.put("date", 202210) - udata.put("avg", r.avg_price.toInteger()) - DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) - }else{ - println r.commId +" 不存在" +ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> + def city = RegionData.formatCity("余姚") + d['D'] = city + if(city){ + d['E'] = RegionData.formatRegion(city, d['C']) } + +// if(!d['D']){ +// def citys = CommonDatautils.getCityByRegion(d['A']) +// if(citys && citys.size()==1){ +// d['F'] = citys.get(0) +// } +// } +}) + +//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1) +//def m = [:] +//for(def d : datas){ +// m.put(d['B'], d) +//} + +//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d-> +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// if(dist){ +// def ret = m.get(dist._id.toString()) +// if(ret){ +// d['E'] = ret['J'] +// d['F'] = ret['N'] +// } + +// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS) +// if(dbo){ +// if(dbo.beike_esf_price){ +// d['G'] = dbo.beike_esf_price +// }else if(dbo.beike_esf_price){ +// d['G'] = dbo.beike_esf_price +// } +// +// if(dbo.maitian_esf_price){ +// d['H'] = dbo.maitian_esf_price +// } +// } +// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd")) +// if(dbo){ +// d['K'] = dbo.avg_price +// d['L'] = dbo.value_type +// } +// +// } +//}) + + + + + +def findThirdPrice(city, region, name, AVERAGE_STATISTICS){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", 202304) + return AVERAGE_STATISTICS.findOne(q) } +//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + +//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") +//def rets = [] +//for(def d : datas){ +// def arrs = d.split("@") +// rets << ["commId":arrs[0], "avg_price":arrs[4]] +//} +// +//for(def r : rets){ +// if(r.avg_price){ +// def udata = new BasicDBObject() +// udata.put("date", 202210) +// udata.put("avg", r.avg_price.toInteger()) +// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) +// }else{ +// println r.commId +" 不存在" +// } +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 8ccb30a9a..a57d7551b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -29,8 +29,11 @@ class AddAvgpriceBatch { } def test(){ - //发重算均价命令 - addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd')) + def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"]) + for(def dist : districts){ + //发重算均价命令 + addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd')) + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index d6432197b..5cae7c343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -96,17 +96,28 @@ class CityStatExport { } def regionStat(){ - def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州'] def ret = [] ret << ["城市","区域","小区总量"] for(def city : citys){ + def regions = CommonDatautils.getRegionListByCity(city) for(def r : regions){ + def row = [] + row << city + row << r def districts = CommonDatautils.findDistricts(city, r, null) println city +" "+ r +" "+ districts.size() + + row << CommonDatautils.queryCityRegionPrice(city, r, null) + def obj = CommonDatautils.findRegionPriceRate(city, r) + row << obj.price_coverage*100 + row << districts.size() + + ret << row } } - ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县") + ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b7d3b371..df4d9454e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -272,6 +272,45 @@ class FiledStrand { return null } + static def processName(name){ + name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期') + if (name.contains('(') && name.contains(')')){ + def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1) + name = name.replace(rep_str, '') + } + + + if (name.contains('(') && name.contains(')')){ + def rep_str = name.subsring(name.index('('),name.rfind(')')+1) + name = name.replace(rep_str, '') + } + + + if (name.find('【')!=-1 && name.find('】')!=-1){ + def rep_str = name[name.index('【'):name.rfind('】')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('【')!=-1 && name.find('】')!=-1){ + def rep_str = name[name.index('【'):name.rfind('】')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('(')!=-1 && name.find(')')!=-1){ + def rep_str = name[name.index('('):name.rfind(')')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('「')!=-1 && name.find('」')!=-1){ + def rep_str = name[name.index('「'):name.rfind('」')+1] + name = name.replace(rep_str, '') + } + return name + } + static def processCarSite(car_site_count){ def v = getValue(car_site_count.toString()); if(v){ @@ -579,6 +618,12 @@ class FiledStrand { }else{ return null; } + }else if(estateType2=="写字楼"){ + if(estateCharege>=0.35 && estateCharege<=60){ + return estateCharege; + }else{ + return null; + } } }catch (Exception e){ println "estateCharege 转不了 "+estateCharege diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index e4e5016ec..7555e8915 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -567,7 +567,7 @@ class OpenApiUtils { } static def financialRating(city, address){ - def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3" + def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" String content = new HttpRequest().get(url, "utf-8"); if(content){ def doc = new JsonSlurper().parseText(content) @@ -580,5 +580,79 @@ class OpenApiUtils { } + //源地址的完整性进行验证 + static def addrVerify(city, address){ + def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + // 省、市、区、街道、社区居委、POI补全 + static def complete(city, address){ + def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + //五级(省/市/区/街道/社区居委)解析、补全 + static def division(city, address){ + def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + //识别地址中省、市、区、道路、POI同时进行地址验证 + static def structure(city, address){ + def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + + /** + * 招商银行测试 + * @param city + * @param address + * @return + */ + static def cmbTest(city, address){ + def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy new file mode 100644 index 000000000..082e2f170 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.gj + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> + if(d['A'] && !d['E']){ +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") + println d['A'] +" "+d['C'] +// def total = findDatas(d['A'], d['B'], d['C']) +// d['D'] = total + def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) + d['E'] = c + } +}) + + + + +def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ d-> + if(d.name.contains(name) && d.type.contains("住宅")){ + total ++ + } + }) + return total; +} + + +def findDatas(def city, def region, def name){ + def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'" + def total = 0 + ElasticUtils.executeSqlQuery(sql, { d-> + total = d.count + }) + return total; +} + +//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d-> +// def region = RegionData.formatCity(d['A']) +// if(!region && d['A']){ +// println d['A'] +// } +//}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy index 73ea03b09..e2eb3f289 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy @@ -76,24 +76,24 @@ MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S -def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") -Calendar calendar = Calendar.getInstance() -calendar.setTime(startDate) -def count = 0 -def hashSet = [] -def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] - -OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> - def result = JSONObject.fromObject(doc.res_map.result) - def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] - data << item - hashSet << result -} - - -//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); -ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") +//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") +//Calendar calendar = Calendar.getInstance() +//calendar.setTime(startDate) +//def count = 0 +//def hashSet = [] +//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +// +//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> +// def result = JSONObject.fromObject(doc.res_map.result) +// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] +// data << item +// hashSet << result +//} +// +// +////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") //def startDate = DateUtil.parseDate("20221201", "yyyyMMdd") @@ -115,4 +115,27 @@ ExcelUtils.write("D:\\ // // ////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); -//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file +//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + + + + +def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230524", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]] + +OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) + def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode, + doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber] + data << item + hashSet << result +} + + +//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy new file mode 100644 index 000000000..9eab92fdb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy @@ -0,0 +1,207 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.util.DateUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 按区域统计覆盖率 + */ + +class XiamenStat { + + public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com") + + public static final def DISTRICT_FIELDS_MAP = [ + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' + ] + + static main(args) { + def task = new XiamenStat(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d-> + if(d['D'] && d['D']=="全市"){ + def dbo = CommonDatautils.findCityInfo(d['C'].replace("市","")) + if(dbo){ + def detail = dbo.district_detail + if(detail){ + d['E'] = dbo.existing_district_count//数据量 + d['F'] = detail.township*100// 乡镇街道 + d['G'] = detail.address*100// 地址 + d['H'] = detail.addrAlias*100// 地址别名 + d['I'] = detail.name*100// 名称 + d['J'] = detail.alias*100// 别名 + d['K'] = detail.alias*100// 映射别名 + d['L'] = detail.lat2*100// 经纬度(百度) + d['M'] = detail.lat*100// 经纬度(高德) + d['N'] = detail.estateType2*100// 物业类型 + d['O'] = detail.buildType*100// 建筑类型 + d['P'] = detail.streetCommunity*100// 街道社区 + d['Q'] = detail.committee*100// 居委会 + d['R'] = detail.businessQuarter*100// 商圈 + d['S'] = detail.schools*100// 对口小学 + d['T'] = detail.kindergartens*100// 对口幼儿园 + d['U'] = detail.loudongCount*100// 楼栋总数 + d['V'] = detail.householdCount*100// 户数 + d['W'] = detail.estateCharge*100// 物业费 + d['X'] = detail.carSiteCount*100// 停车位 + d['Y'] = ""// 电梯 + d['Z'] = detail.propertyLife*100// 产权年限 + d['AA'] = detail.houseProperty*100// 产权性质 + d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分) + d['AC'] = detail.landProperty*100// 小区性质 + d['AD'] = detail.finishBuildingDate*100// 竣工时间 + d['AE'] = detail.openDate*100// 交房时间 + d['AF'] = detail.allArea*100// 占地面积 + d['AG'] = detail.buildArea*100// 建筑面积 + d['AH'] = detail.cubagePercent*100// 容积率 + d['AI'] = detail.virescencePercent*100// 绿化率 + d['AJ'] = detail.mainType*100// 主力户型 + d['AK'] = detail.mainTypeArea*100// 主力户型面积 + d['AL'] = detail.developer*100// 开发商 + d['AM'] = detail.estateCompany*100// 物业公司 + d['AN'] = dbo.price_coverage*100// 小区当前均价 + } + } + }else{ + def fields = DISTRICT_FIELDS_MAP.keySet().asList() + def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields) + def detail = districtDetail(regionDistricts.size(), regionDistricts); + if(detail){ + d['E'] = regionDistricts.size()//数据量 + d['F'] = detail.township*100// 乡镇街道 + d['G'] = detail.address*100// 地址 + d['H'] = detail.addrAlias*100// 地址别名 + d['I'] = detail.name*100// 名称 + d['J'] = detail.alias*100// 别名 + d['K'] = detail.alias*100// 映射别名 + d['L'] = detail.lat2*100// 经纬度(百度) + d['M'] = detail.lat*100// 经纬度(高德) + d['N'] = detail.estateType2*100// 物业类型 + d['O'] = detail.buildType*100// 建筑类型 + d['P'] = detail.streetCommunity*100// 街道社区 + d['Q'] = detail.committee*100// 居委会 + d['R'] = detail.businessQuarter*100// 商圈 + d['S'] = detail.schools*100// 对口小学 + d['T'] = detail.kindergartens*100// 对口幼儿园 + d['U'] = detail.loudongCount*100// 楼栋总数 + d['V'] = detail.householdCount*100// 户数 + d['W'] = detail.estateCharge*100// 物业费 + d['X'] = detail.carSiteCount*100// 停车位 + d['Y'] = ""// 电梯 + d['Z'] = detail.propertyLife*100// 产权年限 + d['AA'] = detail.houseProperty*100// 产权性质 + d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分) + d['AC'] = detail.landProperty*100 // 小区性质 + d['AD'] = detail.finishBuildingDate*100// 竣工时间 + d['AE'] = detail.openDate*100// 交房时间 + d['AF'] = detail.allArea*100// 占地面积 + d['AG'] = detail.buildArea*100// 建筑面积 + d['AH'] = detail.cubagePercent*100// 容积率 + d['AI'] = detail.virescencePercent*100// 绿化率 + d['AJ'] = detail.mainType*100// 主力户型 + d['AK'] = detail.mainTypeArea*100// 主力户型面积 + d['AL'] = detail.developer*100// 开发商 + d['AM'] = detail.estateCompany*100// 物业公司 +// d['AN'] = ""// 小区当前均价 + } + } + }) + } + + + + def districtDetail(def disCount, def regionDistricts) { + + def countMap = [:] + for(def dbo : regionDistricts) { + DISTRICT_FIELDS_MAP.each() {key,skey-> + if (isValid(dbo.get(key))) { + if(countMap.containsKey(key)) { + countMap.put(key, countMap.get(key) + 1) + }else { + countMap.put(key, 1) + } + } + } + } + + def districtDetail = [:]; + DISTRICT_FIELDS_MAP.each() {key,skey-> + if(countMap.containsKey(key)) { + districtDetail.put(skey, roundDouble(countMap.get(key), disCount)) + }else { + districtDetail.put(skey, 0) + } + } + + + if(countMap.containsKey("lat") && countMap.containsKey("lng")) { + districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount)); + }else { + districtDetail.put('latLng', 0); + } + + if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) { + districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount)); + }else { + districtDetail.put('latLng2', 0); + } + return districtDetail; + } + + + def CONTAINS_KWS = ["...", "暂无", "待定", "未知"] + def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."] + def isValid(def value) { + if (!value) { + return false; + } + + if (value instanceof String) { + value = value.trim() + for(String ck : CONTAINS_KWS) { + if(value.contains(ck)) { + return false; + } + } + for(String ek : EQUALS_KWS) { + if(ek.equals(value)) { + return false; + } + } + } else if (value instanceof Number) { + if (value <= 0) { + return false; + } + } else if (value instanceof Collection || value instanceof Map) { + if (value.size() == 0) { + return false; + } + } + return true; + } + + + double roundDouble(int a, int base) { + if(a == 0 || base == 0) { + return 0 + } + return ((double)a / (double)base * 10000.0).toInteger() / 10000.0 + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index a04bae008..9ef0881e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() + def citys = CommonDatautils.findAllCity().reverse() def analysRets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 70249a1f9..db058c5a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -95,7 +95,7 @@ class FangjiaClientUtils { try { if(address){ address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy new file mode 100644 index 000000000..9ec05ed28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 写字楼字段清洗入库(自动入库) + */ +@BatchOwner(Watcher.GaoJuan) +class OfficeFieldCleanBatch extends BaseBatch{ + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + static main(args) { + runBatch(new OfficeFieldCleanBatch(), args) + } + + @Override + void process(Object args) { + def citys = CommonDatautils.findAllCity().reverse() + for(def c : citys){ + def offices = findAllOffice(c) + strandField(offices) + } + + } + + + def strandField(offices){ + def fields = ["property_fee","build_area","","","","","","","","","",""] + for(def office : offices){ + fields.each{ + def newVal = null + if(it == "property_fee"){ + newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼") + newVal = newVal.toString() + } + + if(it =="all_area" || it =="build_area"){ + newVal = FiledStrand.processArea(office.get(it)) + if(newVal && newVal > 5000){ + newVal = newVal.toString() + } + } + + if(it =="name"){ + newVal = FiledStrand.processName(office.get(it)) + if(newVal){ + newVal = newVal.toString() + } + } + } + } + + } + + + def findAllOffice(city){ + def sql = "select * from office where city='${city}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + }) + return rets + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index a59b0747a..c26ded24c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -144,5 +144,8 @@ class SeaweedDataMatchProcess { } + public static void main(def agrs){ + doMatch("上海", "静安", "金家巷西弄", null) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy index aa3c3c62c..6b0f2112c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy @@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250& stat() def stat(){ - def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1) def map=[:] for(def d : datas){ def count = map.get(d['C']) @@ -69,8 +69,8 @@ def findLogs(){ def findFangjiaLogs(){ - def startDate = DateUtil.parseDate("20230329", "yyyyMMdd") - def endDate = DateUtil.parseDate("20230429", "yyyyMMdd") + def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") + def endDate = DateUtil.parseDate("20230531", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def list = [] @@ -84,14 +84,21 @@ def findFangjiaLogs(){ QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false); - def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") +// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") +// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + + def q = QueryBuilders.fuzzyQuery("request", "/enquire") QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); println houses.size() if (houses) { for (def d : houses) { def reqStr = d.request.toString() + if(!reqStr.contains("/enquire")){ + continue + } def mid = null if (reqStr.contains("mid=")) { @@ -107,5 +114,5 @@ def findFangjiaLogs(){ startDate = nextDate } - ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志") + ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志") } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy new file mode 100644 index 000000000..626af0640 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy @@ -0,0 +1,232 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils +import org.apache.lucene.queryparser.classic.QueryParser + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class HouseAnaly { + + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new HouseAnaly() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def allRets = [] + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"]) + def rets = processing(districts) + println "[${city}] 写文件 "+rets.size() + allRets.addAll(rets) + } + ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析") + } + + def processing(dists){ + def rets = new CopyOnWriteArrayList() + rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i=1){ + def ret = analy(dist) + if(ret){ + rets.addAll(ret) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + return rets; + } + + def analy(dist){ + def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是"); + def alias = dist.alias + for(def aliasName : alias){ + if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){ + def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否"); + if(aliaRet){ + masterRet.addAll(aliaRet) + } + } + } + if(masterRet.size()>=2){ + def masterAvgPrice = 0 + def masterMidPrice = 0 + def back = false + for(def row : masterRet){ + if(row.get(4)=="是"){ + masterAvgPrice = row.get(8) + masterMidPrice = row.get(9) + }else{ + def pre1 = 0 + def pre2 = 0 + if(masterAvgPrice>0){ + pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice) + } + if(masterMidPrice>0){ + pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice) + } + if(pre1>=20 || pre2>=20){ + back = true + } + } + } + if(back){ + return masterRet + } + + } + return null; + } + + //小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量 + def runAnalying(id, city, region, name, flag){ + def houses = findHouse(city, region, name); + def rets = [] + if(houses && houses.size()>=5){ + def prices = [] + for(def house : houses){ + prices << house.avg_price + } + prices = prices.sort() + def average = (int)(prices.sum() / prices.size()) + def midPrice = getMidPrice(prices) + def price10 = null + def price90 = null + if(houses.size()>=10){ + def postion = Math.round(houses.size()/10) + price10 = prices.get((int)(postion-1)) + price90 = prices.get((int)(houses.size()-postion-1)) + } + def countMap = classType(houses) + +// def pre = java.lang.Math.abs((midPrice-average)*100/average) +// if(pre>=20){ + rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")] +// } + } + return rets + } + + def classType(houses){ + def district = 0 + def villa = 0 + def business = 0 + for(def h : houses){ + if(h.estate_type){ + if(h.estate_type == "别墅"){ + villa = villa + 1 + continue + }else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){ + business = business + 1 + continue + } + } + def valla = isVilla(h.area, h.name, h.height, h.room) + if(valla){ + villa = villa + 1 + continue + } + district = district + 1 + } + return ["district":district, "villa":villa, "business":business] + } + + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + def getMidPrice(def prices){ + def mid = prices.size()/2 + if(mid > (int)mid){ + return prices.get((int)mid) + }else{ + def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1)) + return (int)(total/2) + } + return null + } + + + def findHouse(def city, def region, def name){ + def houses = [] + try{ + name = escapeExprSpecialWord(name) + String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'" + ElasticUtils.executeSqlQuery(sql,{ d-> + def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type] + houses << r +// println r + }) + }catch(Exception e){ + e.printStackTrace() + } + + return houses + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy index ee95b0621..adc26463e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy @@ -21,13 +21,23 @@ class GaoDeMapSearchTest { static main(args) { def task = new GaoDeMapSearchTest(); - def path = "D:\\数据\\光大\\光大测试-V2.xlsx" + def path = "D:\\数据\\富民\\富民测试500.xlsx" // task.structureV2(path) // task.match(path) // task.mapSearch(path) - task.baiduMapSearch(path) - task.checkTrue(path) +// task.baiduMapSearch(path) +// task.checkTrue(path) + task.addFinancialLevel(path) + } + + def addFinancialLevel(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A']) + if(results){ + row['V'] = results.consumptionRating + } + }); } def checkTrue(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 4961abd56..1068d1f84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + //广州,佛山 重庆 static main(args) { def task = new BeiKeCrawlByChrome(); @@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome { println city +" 已运行" continue } + def newCity = RegionData.formatCity(city) + if(!newCity){ + if(city=="昆山"){ + newCity = "苏州" + } + println city +" 无法格式化" + } def url = m[1] def newUrl = url+"/chengjiao/" driver.get(newUrl) @@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome { def href = url+ r.attr("href") println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, city, href, regionText) + if(city == "昆山"){ + regionText = "昆山" + } + + def detailUrls = getPageDeail(driver, newCity, href, regionText) println url +" 详情链接:"+detailUrls.size() } } @@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e, region) + def dbo = findTrade(durl , "贝壳找房") + if (!dbo) { + parse(city, e, region, driver) + } }else{ stop = true; break; @@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e, region){ + def parse(city,e, region, driver){ try { + def detailUrl = e.select(".title a").attr("href") + + driver.get(detailUrl) + def detailDoc = checkDocument(driver, detailUrl); + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() def arrs = title.split(" ") + if(title.contains("车位")){ + detail.type = "车位" + } detail.city = city detail.district_name = arrs[0] detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) @@ -266,28 +289,42 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" + def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + if(liftStr){ + detail.lift = DataFormatUtils.parseLift(liftStr) + detail.lift_household = DataFormatUtils.parseHousehold(liftStr) + } + detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡","")) if(detail.total_price && detail.avg_price){ - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + detail.url = detailUrl + + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + + detail.floor_info = floorStr + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorInfo && floorInfo.isNumber()) { + detail.height = floorInfo.toInteger() + } } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } }catch(Exception ex){ @@ -392,16 +429,16 @@ public class BeiKeCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def url, source){ def q = new BasicDBObject() // q.put("city", detail.city) // q.put("region", detail.region) // q.put("district_name", detail.district_name) // q.put("area", detail.area) // q.put("total_price", detail.total_price) - q.put("url", detail.url) + q.put("url", url) // q.put("trade_date", detail.trade_date) - q.put("source", detail.source) + q.put("source", source) def dbo = DEAL_PRICE_COLL.findOne(q) return dbo } @@ -446,12 +483,15 @@ public class BeiKeCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(10000) + Thread.sleep(20000) }catch(Exception e){ - Thread.sleep(10000) + Thread.sleep(15000) } return driver } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy new file mode 100644 index 000000000..530579eb8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.paimai + +class DataFormatUtils { + + static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000] + + static def parseLift(def str){ + if(str && str.contains("梯")){ + def lift = str.toString().substring(0, str.toString().indexOf("梯")) + if(lift){ + def val = chineseIntegerPart2Digit(lift) + if(val){ + return (int)val + } + } + } + return null + } + + static def parseHousehold(def str){ + if(str && str.contains("梯") && str.contains("户")){ + def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户")) + def val = chineseIntegerPart2Digit(household) + if(val){ + return (int)val + } + } + return null + } + + + static def chineseIntegerPart2Digit(String chineseDigit) + { + Double finalDigit = 0.0; + Integer r = 1; + + for(int i = chineseDigit.length()-1; i>=0;i--) + { + Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i))); + + if( val >= 10 && i == 0 ) + { + if (val > r) + { + r = val; + finalDigit = finalDigit +val; + } + else + { + r = r * val; + } + } + else if(val >= 10) + { + if (val > r) + { + r = val; + } + else + { + r = r * val; + } + } + else + { + finalDigit = finalDigit + r*val; + } + } + return finalDigit; + } + + static String getValue(String s) { + if (s && s.contains("暂无")) { + return null + } + return s + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index aaf0c9c0d..0c82ca47e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -196,9 +196,22 @@ public class LianjiaCrawlByChrome { println city +" 已运行" continue } + def newCity = RegionData.formatCity(city) + if(!newCity){ + if(city=="昆山"){ + newCity = "苏州" + } + println city +" 无法格式化" + } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" - driver.get(newUrl) + try{ + driver.get(newUrl) + }catch(Exception e){ + println newUrl +" 链接有问题!" + continue + } + //def detailUrls = getPageUrls(driver, cityUrl) Thread.sleep(10000) def doc = checkDocument(driver, newUrl); @@ -208,9 +221,11 @@ public class LianjiaCrawlByChrome { for(def r : regions){ def regionText = r.text() def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") - + if(city == "昆山"){ + regionText = "昆山" + } println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, city, href, regionText) + def detailUrls = getPageDeail(driver, newCity, href, regionText) println cityUrl +" 详情链接:"+detailUrls.size() } } @@ -284,7 +299,10 @@ public class LianjiaCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e, regionText) + def dbo = findTrade(durl , "链家在线") + if (!dbo) { + parse(city, e, regionText, driver) + } }else{ stop = true; break; @@ -302,85 +320,106 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e, regionText){ + def parse(city,e, regionText, driver){ try { - def title = e.select(".title a").text() - if (title) { - def detail = new BasicDBObject() - def arrs = title.split(" ") - detail.city = city - detail.district_name = arrs[0] - if(regionText){ - detail.region = regionText - }else{ - detail.region = findRegionByThirdPary(detail.city, detail.district_name) - } - if(detail.region){ - detail.region = RegionData.formatRegion(city, detail.region) - } - - def roomStr = arrs[1] - if (roomStr) { - if (roomStr.indexOf("室") > 0) { - def room = roomStr.substring(0, roomStr.indexOf("室")) - if(room && room.isNumber()){ - detail.room = room.toInteger() - } + def price = e.select(".totalPrice .number").text() + if(price){ + def detailUrl = e.select(".title a").attr("href") + driver.get(detailUrl) + def detailDoc = checkDocument(driver, detailUrl); + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + if(title.contains("车位")){ + detail.type = "车位" } - if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - if(hall && hall.isNumber()){ - detail.hall = hall.toInteger() - } + detail.city = city + detail.district_name = arrs[0] + if(regionText){ + detail.region = regionText + }else{ + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + } + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) } - } - if("车位"!=roomStr){ - if(arrs && arrs.size()>=3){ - def areaStr = arrs[2].replace("平米", "") - if (areaStr) { - detail.area = areaStr.toDouble() + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + def room = roomStr.substring(0, roomStr.indexOf("室")) + if(room && room.isNumber()){ + detail.room = room.toInteger() + } + + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hall && hall.isNumber()){ + detail.hall = hall.toInteger() + } } } - def price = e.select(".totalPrice .number").text() - if (price && price.isNumber()) { - detail.total_price = (int)(price.toDouble() * 10000) - } + if("车位"!=roomStr){ + if(arrs && arrs.size()>=3){ + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + } - if (detail.total_price && detail.area) { - detail.avg_price = (int) (detail.total_price / detail.area) - } - def tradeStr = e.select(".dealDate").text() - if (tradeStr) { - detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") - } - detail.source = "链家在线" - detail.url = e.select(".title a").attr("href") - if(detail.total_price && detail.avg_price){ - def dbo = findTrade(detail) - if (!dbo) { + if (price && price.isNumber()) { + detail.total_price = (int)(price.toDouble() * 10000) + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + detail.url = detailUrl + + def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + if(liftStr){ + detail.lift = DataFormatUtils.parseLift(liftStr) + detail.lift_household = DataFormatUtils.parseHousehold(liftStr) + } + detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层","")) + + if(detail.total_price && detail.avg_price){ def houseInfo = e.select(".houseInfo").text() def houseInfos = houseInfo.split("\\|") detail.direction = houseInfos[0] detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() + + detail.floor_info = floorStr if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() + def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr2 && floorStr2.isNumber()) { + detail.height = floorStr2.toInteger() } } - detail.create_date = new Date() detail.m_date = new Date() println detail DEAL_PRICE_COLL.save(detail) } } - } } }catch(Exception ex){ @@ -484,16 +523,16 @@ public class LianjiaCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def url, source){ def q = new BasicDBObject() // q.put("city", detail.city) // q.put("region", detail.region) // q.put("district_name", detail.district_name) // q.put("area", detail.area) // q.put("total_price", detail.total_price) + q.put("url", url) // q.put("trade_date", detail.trade_date) - q.put("url", detail.url) - q.put("source", detail.source) + q.put("source", source) def dbo = DEAL_PRICE_COLL.findOne(q) return dbo } @@ -535,7 +574,7 @@ public class LianjiaCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(10000) + Thread.sleep(15000) }catch(Exception e){ Thread.sleep(10000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy new file mode 100644 index 000000000..949e55f51 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.gj.region + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 格式化其它网站区域 + */ + + +def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + +DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl) + + + +def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1) +for(def d : datas){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("recognizeCity", d['F']) + q.put("recognizeRegion", d['A']) + def count = SERVER_AREA_COLL.count(q) + if(!count){ + def obj = new BasicDBObject() + obj.put("city", d['A']) + obj.put("recognizeRegion", d['A']) + obj.put("recognizeCity", d['F']) + obj.put("createdDate", new Date()) + SERVER_AREA_COLL.save(obj) + }else{ + println d['A']+" "+ d['F'] +" 已存在" + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 96709e230..9c6952a70 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -237,7 +237,7 @@ public class DbUtils{ def lng = district.lng / 10000000000.0 sdq.put("sd.point", lat + "," + lng); sdq.put("sd.sfield", "lat_lng"); - sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0)); + sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0)); query.append('$sd', sdq) def keys = new BasicDBObject()