diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy index 6dcae7f53..a11b60d8d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictStatRankBatch.groovy @@ -8,12 +8,19 @@ import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ServerAreaMap import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * Created by dxy on 2016/12/15. */ @@ -23,17 +30,23 @@ class DistrictStatRankBatch extends BaseBatch { private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") private static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def DISTRICT_STAT_MAP = [:] public void process(Object args) { - def cities = ServerAreaMap.findAllCities() + + + def cities = ["深圳"]//ServerAreaMap.findAllCities() for(String city : cities) { - DISTRICT_STAT_MAP.clear() - Map> regions = RegionData.getAllRegions(city) - regions.each {region, alias-> - processRegionRank(city, region) - } - processCityRank(city) +// DISTRICT_STAT_MAP.clear() +// Map> regions = RegionData.getAllRegions(city) +// regions.each {region, alias-> +// processRegionRank(city, region) +// } +// processCityRank(city) + + processCityRentRank(city) } } @@ -257,6 +270,81 @@ class DistrictStatRankBatch extends BaseBatch { return ret } + def processCityRentRank(city){ + def lastMonth = DateUtil.formatByPattern(DateUtil.getLastEndMonthDate(new Date()),"yyyyMMdd") + def distRentsMap = CommonDatautils.findSqmetreRentByCityAndDate(city, lastMonth) + def prices = [] + distRentsMap.each{key, price-> + if(!prices.contains(price)){ + prices << price + } + } +// prices = prices.unique() + prices = prices.sort().reverse() + + def totalCount = new AtomicLong() + def keys = distRentsMap.keySet().asList() + final CountDownLatch latch = new CountDownLatch(keys.size()) + +// for(def i=0;i + index ++ + def arrs = key.split("##"); + double percentile = calculatePercentile(prices, price); + + DBObject q = new BasicDBObject("city", arrs[0]) + q.append("region", arrs[1]) + q.append("district_name", arrs[2]) + + DBObject udata = new BasicDBObject() + udata.put("rent_city_rank", (int)percentile); + udata.put("m_date", new Date()) + DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false) + println index +" "+ arrs[2] + } + } + + def upateDistrict(key, distRentsMap, prices){ + def arrs = key.split("##"); + def price = distRentsMap.get(key) + + double percentile = calculatePercentile(prices, price); + + DBObject q = new BasicDBObject("city", arrs[0]) + q.append("region", arrs[1]) + q.append("district_name", arrs[2]) + + DBObject udata = new BasicDBObject() + udata.put("rent_city_rank", (int)percentile); + udata.put("m_date", new Date()) +// DISTRICT_ANALY_DATA_COLL.update(q, new BasicDBObject('$set', udata), false, false) + } + + public static int calculatePercentile(def values, def targetValue) { + // 找出数字在List中的排名 + def rank = values.indexOf(targetValue) + 1 + // 计算百分位数 + def percentile = (int)((rank / values.size()) * 100) + return percentile; + } static main(args) { runBatch(new DistrictStatRankBatch(), args) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index f34d40421..1c1fa2788 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -51,7 +51,19 @@ for(def d : datas){ map.region = d['B'] def roomStr = d['E'] if(roomStr){ - map.room = roomStr.toString().toDouble().toInteger() + if(roomStr.contains("-")){ + def arrs = roomStr.split("-") + try{ + map.room = arrs[0].toString().toInteger() + }catch(Exception e){ + println "room error" + } + map.hall = arrs[1].toString().toInteger() + map.toilet = arrs[3].toString().toInteger() + }else{ + map.room = roomStr.toString().toDouble().toInteger() + } + } map.area = d['F'].toString().toDouble() @@ -59,7 +71,13 @@ for(def d : datas){ map.avg_price = (int)(d['H'].toString().toDouble()) map.sell_price = (int)(d['I'].toString().toDouble()*10000) def trade_date = d['J'] - map.trade_date = trade_date + if(trade_date){ + map.trade_date = trade_date + }else{ + def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") + } + map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c52b21fcb..42feb5869 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -77,7 +77,6 @@ class BigdataCommunityBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" - delPgData() saveToPgSql(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 6854ae591..249590993 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -66,7 +66,8 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } println "curDate:"+curDate -// for(def city : citys){ +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ //// if(!ccbCitys.contains(city)){ // addDealPriceMatch(city); //// } @@ -642,7 +643,7 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) - def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) + def trades = CcbCommonUtils.findTradesByMonth(city, startDate) final CountDownLatch latch = new CountDownLatch(trades.size()) def final AtomicInteger count_ = new AtomicInteger(0) @@ -665,7 +666,7 @@ class BigdataDealBatch extends BaseBatch{ rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address); } } - if(rMap){ + if(rMap && rMap.region==region){ CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index f4e661538..3479e90ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 465eac28c..3f7cdff5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -428,7 +428,7 @@ class CcbCommonUtils { public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index cdad2a607..1e818b3ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -28,9 +28,11 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); - task.deleteThirdPartyData() +// task.deleteThirdPartyData()//删除第三方 // task.delDealByPe() // task.delDealData() + + task.exportThirdParty() } def delDealByPe(){ @@ -372,15 +374,29 @@ class DealPriceDataProcess { return map } + def exportThirdParty(){ + def citys = CcbCommonUtils.citys_all + def results = [] + results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE'] + for(def city : citys) { + def rets = findPeGteThirdParty3(city, 202305) + for(def map : rets){ + results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe] + } + println city +" "+ rets.size() + } + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\第三方\\202305第三方测试偏差较大案例统计数据.xlsx", results,"第三方") + } def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202303 + def month = 202304 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202304) + def rets = findPeGteThirdParty2(city, 202305) def total = 0 println city +" "+rets.size() if(rets){ @@ -431,6 +447,28 @@ class DealPriceDataProcess { return rets } + //城市id 城市名称 成交id 区域 小区id 小区名称 计算类型 计算价格 成交价格 PPE + + def findPeGteThirdParty3(cityName, yearmonth){ + def sql = "SELECT urbn_id, urbn_nm, gujia_fsid,gujia_rgon_nm, gujia_comm_id,gujia_comm_nm,gujia_bds_clc_tp,gujia_bds_avprc,gujiabds_to_mkprc_mny,gujia_mns_rto from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>15" + def rets = [] + pgsqlInstance.eachRow(sql,{r-> + def map = [:] + map.cityid = r.urbn_id + map.city = r.urbn_nm + map.data_id = r.gujia_fsid + map.region = r.gujia_rgon_nm + map.comm_id = r.gujia_comm_id + map.district_name = r.gujia_comm_nm + map.type = r.gujia_bds_clc_tp + map.clc_price = r.gujia_bds_avprc + map.price = r.gujiabds_to_mkprc_mny + map.ppe = r.gujia_mns_rto + rets << map + }); + return rets + } + def delDealData(){ ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 4df84ea86..8c765a987 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -79,9 +79,9 @@ class ExportOracleAllDataTest { // exportCommCaseDetailGj()// // // ok文件 - exportInOk() - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// exportInOk() +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 5c2e8c26c..60e475036 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -33,7 +33,7 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() -// task.addFjwPrice() + task.addFjwPrice() // task.addCcbPrice()//oracle // task.addCcbPriceFromGuided() @@ -42,7 +42,7 @@ class FjwPricePkCcbPrice { // task.exportFjwPriceAndCcbPrice() // task.addNewDistPrice() - task.addCcbPrice2()//按id查 +// task.addCcbPrice2()//按id查 // task.addCcbPrice3()//按city,region,name查 // task.standRegion() // task.addGuidPrice() @@ -217,17 +217,19 @@ class FjwPricePkCcbPrice { } def addFjwPrice(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d-> - if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['C']) + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\录入人工指导价_0526.xlsx",{d-> + if(!d['N']){ + def dist = CommonDatautils.findDistrictById(d['E']) if(dist){ - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd")) - if(dbo){ - d['K'] = dbo.avg_price - } + d['N'] = dist.region +// def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-05-21","yyyy-MM-dd")) +// if(dbo){ +// d['K'] = dbo.avg_price +// } + + } } - }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 953a53a6b..591eb95bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -83,8 +83,8 @@ class GuiDingPriceBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); saveToPgSql(rows) - - //更新静态数据表 +// +// //更新静态数据表 updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index db8fef0a1..2ce980c49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -33,8 +33,8 @@ import java.util.regex.*; def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -1287,18 +1287,18 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -// -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ -//// def region = RegionData.formatRegion(d['A'], d['B']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") -//// if(dist){ -//// d['E'] = dist.region -//// d['F'] = dist._id.toString() -//// } -//// } +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx" + +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ + def region = RegionData.formatRegion(d['B'], d['C']) + if(region){ + def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") + if(dist){ + d['G'] = dist.region + d['H'] = dist._id.toString() + } + } // if(!d['F']){ // def dist = CommonDatautils.findDistrictById(d['A']) //// @@ -1308,30 +1308,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ // d['F'] = dist.status // } // } -// -// } -//},0) - - -def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") - if(count && count==1){ - def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") - d['E'] = dist.city } - -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['B'] = dist.region -// d['C'] = dist.name -// d['J'] = dist.status -// } -// def newRegion = RegionData.formatRegion(d['A'], d['B']) -// d['F'] = newRegion },0) + + +//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") +// if(count && count==1){ +// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") +// d['E'] = dist.city +// } +// +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['J'] = dist.status +//// } +//// def newRegion = RegionData.formatRegion(d['A'], d['B']) +//// d['F'] = newRegion +//},0) + //ExcelUtils.modify(fileName, {d-> // if(d['A']){ // def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index da8662b0e..492fb152d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); saveToPgSql(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e8bd49824..9156b9d6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -45,6 +45,7 @@ class TradeStatLog { // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() + // task.deleteDistrict() // task.deleteDealData() // task.deleteDealData2() @@ -92,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-02-01" - def date = "2023-03-01" - def endDate = "2023-04-01" - def newDate = "2023-05-01" + def startDate = "2023-03-01" + def date = "2023-04-01" + def endDate = "2023-05-01" + def newDate = "2023-06-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 5a38c237e..9ed8300cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); - cityRun(citys, startDate, endDate, cityCodeMap) - regionRun(citys, startDate, endDate, cityCodeMap) +// cityRun(citys, startDate, endDate, cityCodeMap) +// regionRun(citys, startDate, endDate, cityCodeMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 852f3783a..ae9f4f058 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -33,10 +33,10 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy new file mode 100644 index 000000000..66f7dd391 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FumingTest.groovy @@ -0,0 +1,261 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql + +class FumingTest { + + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + + static main(args) { + def task = new FumingTest(); + task.run(); + } + + def run() { + def path = "D:\\数据\\富民\\富民测试500-v7.xlsx" +// match(path) +// addPrice(path) + addThirdPrice(path) +// addLastPrice(path) +// addFacLevel(path) + } + + + def addFacLevel(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['K'] && row['I']){ +// def obj = findCommunitySummary(row['C']) +// if(obj){ +// row['K'] = obj.consumption_rating +// }else{ + row['K'] = getNearByDistrictLevel(row['C'], row['I']) +// } + } + }) + } + + + /** + * 一公里小区数量 + */ + def getNearByDistrictLevel(def id, def price){ + BasicDBObject district = DbUtils.seaweed.findById(id); + def level = null + if(district){ + List list = DbUtils.findAroundDistrict(district); + println id +" 共:"+list.size()+" "+ price + if(list!=null&&list.size()>0){ + for(def obj : list){ + def summary = findCommunitySummary(obj._id.toString()) + if(summary && summary.avg_price && summary.consumption_rating){ + def pre = java.lang.Math.abs((summary.avg_price - price)*100/price) + println summary.name +" "+pre +" "+ summary.avg_price + if(pre <= 10){ + level = summary.consumption_rating + break; + } + } + } + } + } + return level; + } + + + def addLastPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Z'] ){ + if(row['W']){ + row['Z'] = row['W'] + }else if(row['R']){ + row['Z'] = row['R'] + }else if(row['I']){ + row['Z'] = row['I'] + } + + if(row['Z']){ + row['J'] = (int)(row['Z']*row['B'].toString().toDouble()) + } + } + }) + } + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['N'] ){ + def city = row['C'] + def address = row['A'] + def region = row['D'] + def name = row['G'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) + if(jsonObject != null && jsonObject.precise!=-1) { + row['N'] = jsonObject.get("id") + row['O'] = jsonObject.get("city") + row['P'] = jsonObject.get("district") + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("propertyType") + row['S'] = jsonObject.get("address") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['C'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + row['D'] = dist.city + row['E'] = dist.region + row['F'] = dist.name + row['G'] = dist.estate_type2 + row['H'] = dist.address +// row['Q'] = dist.name + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['I'] = priceObj.avg_price + row['J'] = (int)(priceObj.avg_price*row['B'].toString().toDouble()) + row['R'] = priceObj.avg_price_2b + } + } + } + }) + } + + + def addThirdPrice(path){ +// def citys = ['昆明','廊坊','东莞','西安','南宁','成都','佛山','广州','重庆','珠海','长沙','中山','张家口','武汉','苏州','南通','贵阳','天津','大连'] + def allMap = [:] +// for(def c : citys){ +// def cityMap = findThirdPartyPrice(c) +// allMap << cityMap +// } + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F']) + def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['L'] = priceMap.get("lianjia_esf_price") + } + + if(priceMap.containsKey("beike_esf_price")){ + row['M'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("anjuke_esf_price")){ + row['N'] = priceMap.get("anjuke_esf_price") + } + } + def map = findAvgPrice(row['C'], cityCodeMap.get(row['D'])) + if(map){ + row['O'] = map.avg_price + row['P'] = map.credit_value + } + }) + } + + + + def addCcbPrice2(){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d -> + if(d['C']){ + def map = findAvgPrice(d['C'], cityCodeMap.get(d['E'])) + if(map){ + d['K'] = map.avg_price + d['L'] = map.calc_type +// d['AI'] = map.credit_value + d['M'] = map.month + } + } + }) + } + + def findThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + } + return map + } + + def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + + + def findCommunitySummary(id){ + def obj = null + def sql = "select * from community_summary where id='${id}'" + ElasticUtils.executeSqlQuery(sql, { + obj = it + }) + return obj + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index 2b163d322..cdad0ddb6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -40,19 +40,25 @@ class HailanTest { static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new HailanTest(); task.run(); } def run(){ - def inputfile = "D:\\数据\\海南\\统计.xlsx" + def inputfile = "D:\\数据\\微元素\\排查表.xlsx" +// addThirdPrice(inputfile) // match(inputfile) +// tradeAvgPrice(inputfile) +// caluateTotalPrice(inputfile) // addPrice(inputfile) // addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") // addHouse(inputfile) // stat(inputfile) - hailanStat(inputfile) +// hailanStat(inputfile) + addFzFjwPrice(inputfile) } def addHouse(inputfile){ @@ -533,13 +539,13 @@ class HailanTest { def match(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['C']){ - String city = row['H']//RegionData.formatCity(row['B']) + if(row['A']){ + String city = "福州"//RegionData.formatCity(row['B']) if(city){ - def address = row['C'] + def address = row['A'] // def region = row['C'] println "[match] city:"+city+" address:"+address - def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E']) + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null) if(jsonObject != null && jsonObject.precise!=-1) { row['I'] = jsonObject.get("district") row['J'] = jsonObject.get("name") @@ -554,6 +560,8 @@ class HailanTest { }) } + + def stat(path){ ExcelUtils.modify(path, { row -> if(row['L'] && !row['H']){ @@ -700,4 +708,60 @@ class HailanTest { } return prices; } + + def tradeAvgPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){ + def dist = CommonDatautils.findDistrictById(row['I']) + if(dist){ + def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01"); + row['J'] = (int)tradeAvgPrice + } + } + }) + } + + + def addThirdPrice(path){ + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate) + def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95) + } + } + }) + } + + def caluateTotalPrice(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['J']){ + row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble()) + }else if(row['K']){ + row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble()) + } + }) + } + + def addFzFjwPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['B']) { + def q = new BasicDBObject() + q.put("city", "福州") + q.put("region", row['A']) + q.put("name", row['B']) + q.put("month", 202304) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if (obj) { + row['D'] = obj.avg_price + } + } + + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy index 112de372f..50db279aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -26,7 +26,7 @@ class MingShenPocTest { def run(){ def path = "D:\\数据\\民生\\poc测试.xlsx" //9级标准化 - //structureV2(path)//针对海南poc地址数据此接口不合适使用 + structureV2(path)//针对海南poc地址数据此接口不合适使用 //省市区拆分 // ner(path)//城市,区域带市带区 // match(path) @@ -34,7 +34,7 @@ class MingShenPocTest { // addLatLng(path) // addAroundPrice(path) // addAuthor(path) - addHouse(path) +// addHouse(path) } def structureV2(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/RuiFenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/RuiFenPocTest.groovy new file mode 100644 index 000000000..2d862b119 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/RuiFenPocTest.groovy @@ -0,0 +1,197 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +class RuiFenPocTest { + + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + + static main(args) { + def task = new RuiFenPocTest(); + task.run(); + } + + def run(){ + def inputfile = "D:\\数据\\瑞丰\\房产信息-地址V3-1.xlsx" +// match(inputfile) +// addValue(inputfile) + +// addThirdPrice(inputfile) + lastPrice(inputfile) + } + + + def lastPrice(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['X'] && row['S']){ + def mod = java.lang.Math.abs((row['S'].toString().toDouble()-row['H'].toString().toDouble())*100/row['H'].toString().toDouble()) + if(mod <= 10){ + row['U'] = row['H'] + }else{ + row['U'] = row['S'] + } + }else{ + if(row['K']){ + row['U'] = row['K'] + }else{ + row['U'] = row['H'] + } + + } + }) + } + + def addThirdPrice(path){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = findThirdPartyPrice(row['D'], row['E'], row['F']) + def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['L'] = priceMap.get("lianjia_esf_price") + } + + if(priceMap.containsKey("beike_esf_price")){ + row['M'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("anjuke_esf_price")){ + row['N'] = priceMap.get("anjuke_esf_price") + } + } + def map = findAvgPrice(row['I'], cityCodeMap.get(row['D'])) + if(map){ + row['O'] = map.avg_price + row['P'] = map.credit_value + } + }) + } + + + + def addValue(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + def dist = CommonDatautils.findDistrictById(row['I']) + if(dist){ + row["D"] = dist.city + row["E"] = dist.region + row["F"] = dist.name + row["G"] = dist.address + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['H'] = priceObj.avg_price + row['K'] = priceObj.avg_price_2b + } + } + }) + } + + + def match(path) { + ExcelUtils.multiThreadingModify(path, { row -> + def city = "绍兴" + if (row['B'] && !row["D"]) { + def address = row['B'] + println "[match] city:" + city + " address:" + address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, null) + + if(!row["D"]){ + if (jsonObject != null && jsonObject.precise != -1) { + row["D"] = jsonObject.city + row["E"] = jsonObject.district + row["F"] = jsonObject.name + row["G"] = jsonObject.address + row["I"] = jsonObject.id + def priceObj = CommonDatautils.findDistrictPrice(jsonObject.city, jsonObject.district, jsonObject.name) + if(priceObj){ + row['H'] = priceObj.avg_price + } + } + }else { + if (jsonObject != null && jsonObject.precise != -1) { + if(row['J']){ + def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id + row['K'] = str + }else{ + def str = jsonObject.city + " " + jsonObject.district + " " + jsonObject.name + " " + jsonObject.address + " " + jsonObject.id + row['J'] = str + } + } + } + } + }) + } + + def findThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + } + return map + } + + def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 543973cfa..e8cc8e9f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,11 +1,14 @@ package com.fangjia.script.commData +import com.fangjia.core.RegionData import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import groovy.json.JsonSlurper import net.sf.json.JSONArray import net.sf.json.JSONObject - +import org.apache.commons.lang.StringUtils import org.bson.types.ObjectId import com.fangjia.core.util.HttpRequest; @@ -23,6 +26,12 @@ import com.mongodb.DBCollection import com.mongodb.DBCursor import com.mongodb.DBObject import com.mongodb.QueryOperators +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.regex.Matcher +import java.util.regex.Pattern class CommonDatautils { @@ -73,6 +82,19 @@ class CommonDatautils { public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl) public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo); + public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -230,6 +252,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); +// q.put("name", "长滩美墅三期") DBObject keys = new BasicDBObject(); if(fields){ @@ -781,7 +804,7 @@ class CommonDatautils { public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); q.put("region", new BasicDBObject('$in', regions)) @@ -1001,7 +1024,7 @@ class CommonDatautils { q.put("cat", cat); q.put("region", region) q.put("name", name); - q.put("status", 0) + q.put("visible", 0) DBObject keys = new BasicDBObject(); if(fields){ @@ -3418,11 +3441,27 @@ class CommonDatautils { dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def sqmetreRentMap = [:] for(def d : dbcursor){ - sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) +// sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) + sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.precise_avg_price) } return sqmetreRentMap; } + + static def findAllSqmetreRentPrices(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("cat","district") + q.put("weekend", weekend) + def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000) + dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def prices = [] + for(def d : dbcursor){ + prices << d.avg_price + } + return prices; + } + static def findNewSqmetreRent(def city, def region, def name){ def q = new BasicDBObject() q.put("city", city) @@ -3490,5 +3529,373 @@ class CommonDatautils { } return obj } - + + static def findDistrictAnalyStat(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + def analyData = DISTRICT_ANALY_DATA_COLL.findOne(q) + return analyData + } + + static def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)) + .must(date_range).must(area_range) + if(cat){ + queryBuilder.must(QueryBuilders.termQuery("cat", cat)) + } + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + static def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + + static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.beike_esf_price) + } + } + return map + } + + + /** + * 根据经纬度获取区域 + * @param city + * @param location + * @return + */ + static def getRegionByLocation(city, def location){ + def json = GaoDeApiUtils.geocode(location); + def region = null; + if (json && json.info=="OK") { + def regeocode = json.regeocode + if(regeocode && regeocode.addressComponent){ + region = regeocode.addressComponent.district + if(!region){ + region = regeocode.addressComponent.township + } + } + } + if(region){ + region = RegionData.formatRegion(city, region) + } + return region; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy new file mode 100644 index 000000000..b604c38ff --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.mongodb.BasicDBObject + +class MasterNameInAlias { + + static main(args) { + def task = new MasterNameInAlias() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def allList = [] + for(def city : citys){ + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"]) + + def cityList = [] + cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"] + for(def dist1 : disticts){ + def distList = [] + for(def dist2 : disticts){ + if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){ + if(dist2.alias && dist2.alias.contains(dist1.name)){ + def distance = '' + if(dist1.lng && dist2.lng){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d); + } + distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance] + dist2.alias.remove(dist1.name) + def d = new BasicDBObject() + d.put("_id", dist2._id) + + def udata = new BasicDBObject() + udata.put("alias", dist2.alias) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) + } + } + } + if(distList){ + cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', ''] + cityList.addAll(distList) +// cityList << [] + } + } + + if(cityList){ + allList.addAll(cityList) + println city +" "+cityList.size() + } + } +// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy index 5c1e8c152..58a433b62 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy @@ -35,7 +35,7 @@ class FjtResponseTimeAnaly extends BaseBatch{ @Override public void process(Object args) { //run(); - jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了?") + jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了2?") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index ea6d2717b..75ff94eee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.lang.StringUtils import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -82,20 +83,21 @@ class DistrictInfoExportBatch extends BaseBatch{ } } def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - for(def dist : removeDists){ - def key = dist.city +"##"+dist.region+"##"+dist.name - if(!distKeys.contains(key)){ - distKeys << key - districts << dist - } - } +// for(def dist : removeDists){ +// def key = dist.city +"##"+dist.region+"##"+dist.name +// if(!distKeys.contains(key)){ +// distKeys << key +// districts << dist +// } +// } def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 - saveOrUpdateFzDist(districts) +// saveOrUpdateFzDist(districts) + remarkRemoveDist(removeDists) def fzDistricts = getFzDistricts(city) for(def rMap : fzDistricts){ @@ -185,6 +187,15 @@ class DistrictInfoExportBatch extends BaseBatch{ } + def remarkRemoveDist(removeDists){ + for(def rd : removeDists){ + def udate = new BasicDBObject() + udate.put("status",1) + udate.put("m_date", new Date()) + FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate)); + } + } + def getFzDistricts(def city){ def q = new BasicDBObject() q.put("city", city) @@ -311,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def s = str.toString(); index++; if(s && s!="null"){ + s = s.replace('"','\\"') // if(s.toString().contains(",")){ // s = csvHandlerStr(s.toString()) // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 96ed55fcb..dc389ffe5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -143,13 +143,15 @@ class DistrictSellStatBatch extends BaseBatch { if(weekendPriceObj.value_type!=1){ if(dist.city == "福州"){ if(Math.abs(mom)*100>3){ - def randMarkup = nextDouble(0, 3) - if(mom < 0){ - mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) - }else{ - mom = DataUtil.roundDouble(randMarkup/100, 4) + if(dist.region != "马尾"){ + def randMarkup = nextDouble(0, 3) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100, 4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) } - curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) } }else{ if(Math.abs(mom)*100>10){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 1dd114d8a..76925db82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{ // sb.append(s+ "@|@") // } // } - sb.append(StringUtils.join(row.toArray(), "@|@")) + def str = StringUtils.join(row.toArray(), "@|@") + str = str.replace('"','\\"') + sb.append(str) + sb.append(System.getProperty("line.separator")); println index diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 47a3b331f..9cea6cf74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.core.RegionData import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils @@ -11,26 +12,91 @@ import com.mongodb.DBCollection import org.apache.commons.io.FileUtils import org.bson.types.ObjectId +DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") -DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") -def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") -def rets = [] -for(def d : datas){ - def arrs = d.split("@") - rets << ["commId":arrs[0], "avg_price":arrs[4]] -} - -for(def r : rets){ - if(r.avg_price){ - def udata = new BasicDBObject() - udata.put("date", 202210) - udata.put("avg", r.avg_price.toInteger()) - DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) - }else{ - println r.commId +" 不存在" +ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> + def city = RegionData.formatCity("余姚") + d['D'] = city + if(city){ + d['E'] = RegionData.formatRegion(city, d['C']) } + +// if(!d['D']){ +// def citys = CommonDatautils.getCityByRegion(d['A']) +// if(citys && citys.size()==1){ +// d['F'] = citys.get(0) +// } +// } +}) + +//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1) +//def m = [:] +//for(def d : datas){ +// m.put(d['B'], d) +//} + +//ExcelUtils.modify("D:\\数据\\福州\\202305月福州统计数据\\福州马尾.xlsx",{d-> +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// if(dist){ +// def ret = m.get(dist._id.toString()) +// if(ret){ +// d['E'] = ret['J'] +// d['F'] = ret['N'] +// } + +// def dbo = findThirdPrice(d['A'], d['B'], d['C'], AVERAGE_STATISTICS) +// if(dbo){ +// if(dbo.beike_esf_price){ +// d['G'] = dbo.beike_esf_price +// }else if(dbo.beike_esf_price){ +// d['G'] = dbo.beike_esf_price +// } +// +// if(dbo.maitian_esf_price){ +// d['H'] = dbo.maitian_esf_price +// } +// } +// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], DateUtils.parseFromTightByPattern("2023-05-21", "yyyy-MM-dd")) +// if(dbo){ +// d['K'] = dbo.avg_price +// d['L'] = dbo.value_type +// } +// +// } +//}) + + + + + +def findThirdPrice(city, region, name, AVERAGE_STATISTICS){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", 202304) + return AVERAGE_STATISTICS.findOne(q) } +//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + +//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") +//def rets = [] +//for(def d : datas){ +// def arrs = d.split("@") +// rets << ["commId":arrs[0], "avg_price":arrs[4]] +//} +// +//for(def r : rets){ +// if(r.avg_price){ +// def udata = new BasicDBObject() +// udata.put("date", 202210) +// udata.put("avg", r.avg_price.toInteger()) +// DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) +// }else{ +// println r.commId +" 不存在" +// } +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index a3be166aa..63211976b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 8ccb30a9a..a57d7551b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -29,8 +29,11 @@ class AddAvgpriceBatch { } def test(){ - //发重算均价命令 - addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd')) + def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"]) + for(def dist : districts){ + //发重算均价命令 + addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd')) + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index d6432197b..5cae7c343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -96,17 +96,28 @@ class CityStatExport { } def regionStat(){ - def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = ['淮南','阜阳','黄山','亳州','安庆','六安','淮北','宣城','池州','铜陵','滁州','蚌埠','合肥','芜湖','马鞍山','宿州'] def ret = [] ret << ["城市","区域","小区总量"] for(def city : citys){ + def regions = CommonDatautils.getRegionListByCity(city) for(def r : regions){ + def row = [] + row << city + row << r def districts = CommonDatautils.findDistricts(city, r, null) println city +" "+ r +" "+ districts.size() + + row << CommonDatautils.queryCityRegionPrice(city, r, null) + def obj = CommonDatautils.findRegionPriceRate(city, r) + row << obj.price_coverage*100 + row << districts.size() + + ret << row } } - ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"区县") + ExcelUtils.write("D:\\数据\\安徽省城市统计(楼盘量).xlsx",ret,"区县") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b7d3b371..df4d9454e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -272,6 +272,45 @@ class FiledStrand { return null } + static def processName(name){ + name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期') + if (name.contains('(') && name.contains(')')){ + def rep_str = name.subsring(name.index('('),name.lastIndex(')')+1) + name = name.replace(rep_str, '') + } + + + if (name.contains('(') && name.contains(')')){ + def rep_str = name.subsring(name.index('('),name.rfind(')')+1) + name = name.replace(rep_str, '') + } + + + if (name.find('【')!=-1 && name.find('】')!=-1){ + def rep_str = name[name.index('【'):name.rfind('】')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('【')!=-1 && name.find('】')!=-1){ + def rep_str = name[name.index('【'):name.rfind('】')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('(')!=-1 && name.find(')')!=-1){ + def rep_str = name[name.index('('):name.rfind(')')+1] + name = name.replace(rep_str, '') + } + + + if (name.find('「')!=-1 && name.find('」')!=-1){ + def rep_str = name[name.index('「'):name.rfind('」')+1] + name = name.replace(rep_str, '') + } + return name + } + static def processCarSite(car_site_count){ def v = getValue(car_site_count.toString()); if(v){ @@ -579,6 +618,12 @@ class FiledStrand { }else{ return null; } + }else if(estateType2=="写字楼"){ + if(estateCharege>=0.35 && estateCharege<=60){ + return estateCharege; + }else{ + return null; + } } }catch (Exception e){ println "estateCharege 转不了 "+estateCharege diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index e4e5016ec..7555e8915 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -567,7 +567,7 @@ class OpenApiUtils { } static def financialRating(city, address){ - def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3" + def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" String content = new HttpRequest().get(url, "utf-8"); if(content){ def doc = new JsonSlurper().parseText(content) @@ -580,5 +580,79 @@ class OpenApiUtils { } + //源地址的完整性进行验证 + static def addrVerify(city, address){ + def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + // 省、市、区、街道、社区居委、POI补全 + static def complete(city, address){ + def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + //五级(省/市/区/街道/社区居委)解析、补全 + static def division(city, address){ + def url = "http://open.fangjia.com/addrSvc/division?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + //识别地址中省、市、区、道路、POI同时进行地址验证 + static def structure(city, address){ + def url = "http://open.fangjia.com/addrSvc/structure?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } + + + /** + * 招商银行测试 + * @param city + * @param address + * @return + */ + static def cmbTest(city, address){ + def url = "http://open.fangjia.com/address/cmbTest?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==0 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy index c5ee6bb09..4d5119c0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj +import com.fangjia.core.RegionData import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils @@ -69,29 +70,43 @@ import com.mongodb.DBObject // // //}) -println "开始统计" -def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1) -def map = [:] -for(def d : datas){ - def prices = map.get(d['B']) - if(!prices){ - prices = [] - map.put(d['B'], prices) - } - prices << d['G'] -} - -def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"] -def rows = [] -rows << ["区域","数量", "均价"] - -regions.each{e-> - def prices = map.get(e) - def price = (prices.sum()/prices.size()).toInteger() - rows << [e, prices.size(), price] -} -//map.each{region,prices-> -// def price = (prices.sum()/prices.size()).toInteger() -// rows << [region, prices.size(), price] +//println "开始统计" +//def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1) +//def map = [:] +//for(def d : datas){ +// def prices = map.get(d['B']) +// if(!prices){ +// prices = [] +// map.put(d['B'], prices) +// } +// prices << d['G'] //} -ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果") \ No newline at end of file +// +//def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"] +//def rows = [] +//rows << ["区域","数量", "均价"] +// +//regions.each{e-> +// def prices = map.get(e) +// def price = (prices.sum()/prices.size()).toInteger() +// rows << [e, prices.size(), price] +//} +////map.each{region,prices-> +//// def price = (prices.sum()/prices.size()).toInteger() +//// rows << [region, prices.size(), price] +////} +//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果") + +//ExcelUtils.multiThreadingModify("D:\\数据\\其它\\城市等级.xlsx",{d-> +// d['C'] = RegionData.formatCity(d['A']) +//}) + +DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo); +ExcelUtils.modify("D:\\数据\\其它\\城市等级.xlsx",{d-> + def q = new BasicDBObject() + q.put("display", d['C']) + + def udata = new BasicDBObject() + udata.put("new_city_grade",d['E'].toString().toDouble().toInteger()) + SERVER_AREA_CONN.update(q, new BasicDBObject('$set', udata)) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy new file mode 100644 index 000000000..082e2f170 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.gj + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> + if(d['A'] && !d['E']){ +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") + println d['A'] +" "+d['C'] +// def total = findDatas(d['A'], d['B'], d['C']) +// d['D'] = total + def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) + d['E'] = c + } +}) + + + + +def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ d-> + if(d.name.contains(name) && d.type.contains("住宅")){ + total ++ + } + }) + return total; +} + + +def findDatas(def city, def region, def name){ + def sql = "select count(*) as count from community where city='${city}' and region='${region}' and name='${name}'" + def total = 0 + ElasticUtils.executeSqlQuery(sql, { d-> + total = d.count + }) + return total; +} + +//ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d-> +// def region = RegionData.formatCity(d['A']) +// if(!region && d['A']){ +// println d['A'] +// } +//}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 16cae9383..4de90b96b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-04-01" + static def statDate = "2023-05-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy index 73ea03b09..e2eb3f289 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy @@ -76,24 +76,24 @@ MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.S -def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") -Calendar calendar = Calendar.getInstance() -calendar.setTime(startDate) -def count = 0 -def hashSet = [] -def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] - -OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> - def result = JSONObject.fromObject(doc.res_map.result) - def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] - data << item - hashSet << result -} - - -//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); -ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") +//def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +//def endDate = DateUtil.parseDate("20230508", "yyyyMMdd") +//Calendar calendar = Calendar.getInstance() +//calendar.setTime(startDate) +//def count = 0 +//def hashSet = [] +//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +// +//OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> +// def result = JSONObject.fromObject(doc.res_map.result) +// def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] +// data << item +// hashSet << result +//} +// +// +////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +//ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志") //def startDate = DateUtil.parseDate("20221201", "yyyyMMdd") @@ -115,4 +115,27 @@ ExcelUtils.write("D:\\ // // ////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); -//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file +//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + + + + +def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230524", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [[ "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg","访问时长","parameter_map.showCode", "参数-城市", "参数-地址","省","省编码","城市","城市编码","区","区编码","镇","镇","街道","社区","社区编码","poi","楼栋号","单元号","经度","纬度","格式化地址","有效性","路号","楼层","室号"]] + +OPEN_URI_LOG.find(and(eq("user_id", "1007780"), eq("request_uri","/v2/address/standard"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) + def item = [ "福建顶点软件", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg,doc.time,doc.parameter_map.showCode, + doc.parameter_map.city,doc.parameter_map.address, result.province,result.provinceCode,result.city,result.cityCode,result.district,result.districtCode,result.township,result.townshipCode,result.street,result.community,result.communityCode,result.name,result.buildingNumber,result.unitNumber,result.lng,result.lat,result.formatAddress,result.addrVerify,result.streetNumber,result.floor,result.roomNumber] + data << item + hashSet << result +} + + +//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false); +ExcelUtils.write("D:\\福建顶点软件${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy new file mode 100644 index 000000000..9eab92fdb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/XiamenStat.groovy @@ -0,0 +1,207 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.util.DateUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 按区域统计覆盖率 + */ + +class XiamenStat { + + public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com") + + public static final def DISTRICT_FIELDS_MAP = [ + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' + ] + + static main(args) { + def task = new XiamenStat(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\厦门农商金融\\厦门银行统计1.xlsx",{d-> + if(d['D'] && d['D']=="全市"){ + def dbo = CommonDatautils.findCityInfo(d['C'].replace("市","")) + if(dbo){ + def detail = dbo.district_detail + if(detail){ + d['E'] = dbo.existing_district_count//数据量 + d['F'] = detail.township*100// 乡镇街道 + d['G'] = detail.address*100// 地址 + d['H'] = detail.addrAlias*100// 地址别名 + d['I'] = detail.name*100// 名称 + d['J'] = detail.alias*100// 别名 + d['K'] = detail.alias*100// 映射别名 + d['L'] = detail.lat2*100// 经纬度(百度) + d['M'] = detail.lat*100// 经纬度(高德) + d['N'] = detail.estateType2*100// 物业类型 + d['O'] = detail.buildType*100// 建筑类型 + d['P'] = detail.streetCommunity*100// 街道社区 + d['Q'] = detail.committee*100// 居委会 + d['R'] = detail.businessQuarter*100// 商圈 + d['S'] = detail.schools*100// 对口小学 + d['T'] = detail.kindergartens*100// 对口幼儿园 + d['U'] = detail.loudongCount*100// 楼栋总数 + d['V'] = detail.householdCount*100// 户数 + d['W'] = detail.estateCharge*100// 物业费 + d['X'] = detail.carSiteCount*100// 停车位 + d['Y'] = ""// 电梯 + d['Z'] = detail.propertyLife*100// 产权年限 + d['AA'] = detail.houseProperty*100// 产权性质 + d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分) + d['AC'] = detail.landProperty*100// 小区性质 + d['AD'] = detail.finishBuildingDate*100// 竣工时间 + d['AE'] = detail.openDate*100// 交房时间 + d['AF'] = detail.allArea*100// 占地面积 + d['AG'] = detail.buildArea*100// 建筑面积 + d['AH'] = detail.cubagePercent*100// 容积率 + d['AI'] = detail.virescencePercent*100// 绿化率 + d['AJ'] = detail.mainType*100// 主力户型 + d['AK'] = detail.mainTypeArea*100// 主力户型面积 + d['AL'] = detail.developer*100// 开发商 + d['AM'] = detail.estateCompany*100// 物业公司 + d['AN'] = dbo.price_coverage*100// 小区当前均价 + } + } + }else{ + def fields = DISTRICT_FIELDS_MAP.keySet().asList() + def regionDistricts = CommonDatautils.findDistrictByCityAndRegions(d['C'].replace("市",""), "district", [d['D'].replace("区","")], fields) + def detail = districtDetail(regionDistricts.size(), regionDistricts); + if(detail){ + d['E'] = regionDistricts.size()//数据量 + d['F'] = detail.township*100// 乡镇街道 + d['G'] = detail.address*100// 地址 + d['H'] = detail.addrAlias*100// 地址别名 + d['I'] = detail.name*100// 名称 + d['J'] = detail.alias*100// 别名 + d['K'] = detail.alias*100// 映射别名 + d['L'] = detail.lat2*100// 经纬度(百度) + d['M'] = detail.lat*100// 经纬度(高德) + d['N'] = detail.estateType2*100// 物业类型 + d['O'] = detail.buildType*100// 建筑类型 + d['P'] = detail.streetCommunity*100// 街道社区 + d['Q'] = detail.committee*100// 居委会 + d['R'] = detail.businessQuarter*100// 商圈 + d['S'] = detail.schools*100// 对口小学 + d['T'] = detail.kindergartens*100// 对口幼儿园 + d['U'] = detail.loudongCount*100// 楼栋总数 + d['V'] = detail.householdCount*100// 户数 + d['W'] = detail.estateCharge*100// 物业费 + d['X'] = detail.carSiteCount*100// 停车位 + d['Y'] = ""// 电梯 + d['Z'] = detail.propertyLife*100// 产权年限 + d['AA'] = detail.houseProperty*100// 产权性质 + d['AB'] = detail.buildStructure*100// 建筑结构(按材料划分) + d['AC'] = detail.landProperty*100 // 小区性质 + d['AD'] = detail.finishBuildingDate*100// 竣工时间 + d['AE'] = detail.openDate*100// 交房时间 + d['AF'] = detail.allArea*100// 占地面积 + d['AG'] = detail.buildArea*100// 建筑面积 + d['AH'] = detail.cubagePercent*100// 容积率 + d['AI'] = detail.virescencePercent*100// 绿化率 + d['AJ'] = detail.mainType*100// 主力户型 + d['AK'] = detail.mainTypeArea*100// 主力户型面积 + d['AL'] = detail.developer*100// 开发商 + d['AM'] = detail.estateCompany*100// 物业公司 +// d['AN'] = ""// 小区当前均价 + } + } + }) + } + + + + def districtDetail(def disCount, def regionDistricts) { + + def countMap = [:] + for(def dbo : regionDistricts) { + DISTRICT_FIELDS_MAP.each() {key,skey-> + if (isValid(dbo.get(key))) { + if(countMap.containsKey(key)) { + countMap.put(key, countMap.get(key) + 1) + }else { + countMap.put(key, 1) + } + } + } + } + + def districtDetail = [:]; + DISTRICT_FIELDS_MAP.each() {key,skey-> + if(countMap.containsKey(key)) { + districtDetail.put(skey, roundDouble(countMap.get(key), disCount)) + }else { + districtDetail.put(skey, 0) + } + } + + + if(countMap.containsKey("lat") && countMap.containsKey("lng")) { + districtDetail.put('latLng', roundDouble((countMap.lat > countMap.lng ? countMap.lat : countMap.lng), disCount)); + }else { + districtDetail.put('latLng', 0); + } + + if(countMap.containsKey("lat2") && countMap.containsKey("lng2")) { + districtDetail.put('latLng2', roundDouble((countMap.lat2 > countMap.lng2 ? countMap.lat2 : countMap.lng2), disCount)); + }else { + districtDetail.put('latLng2', 0); + } + return districtDetail; + } + + + def CONTAINS_KWS = ["...", "暂无", "待定", "未知"] + def EQUALS_KWS = ["0.0", "-", "0.00", "0", "."] + def isValid(def value) { + if (!value) { + return false; + } + + if (value instanceof String) { + value = value.trim() + for(String ck : CONTAINS_KWS) { + if(value.contains(ck)) { + return false; + } + } + for(String ek : EQUALS_KWS) { + if(ek.equals(value)) { + return false; + } + } + } else if (value instanceof Number) { + if (value <= 0) { + return false; + } + } else if (value instanceof Collection || value instanceof Map) { + if (value.size() == 0) { + return false; + } + } + return true; + } + + + double roundDouble(int a, int base) { + if(a == 0 || base == 0) { + return 0 + } + return ((double)a / (double)base * 10000.0).toInteger() / 10000.0 + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy new file mode 100644 index 000000000..cc9c87682 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy @@ -0,0 +1,117 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 通过房源更改电梯 + */ + +@BatchOwner(Watcher.GaoJuan) +class AddDistrictElevatorBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + + + static main(args) { + runBatch(new AddDistrictElevatorBatch(), args) + } + + + @Override + void process(Object args) { + run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def dists = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","alias", "build_type", "estate_type2", "is_elevator"]) + addEvevatorProcess(c, dists) + } + } + + def addEvevatorProcess(c, dists){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + println "小区量:"+dists.size() + + def rets = new CopyOnWriteArrayList() + for(def i=0;i=3 && newHouses.size()>=haveHouses.size()*0.6){ + println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size() + BasicDBObject udata = new BasicDBObject() + if(dist.estate_type2 == "别墅"){ + udata.put("is_elevator", false) + }else{ + udata.put("is_elevator", true) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata)) + rets << dist + }else{ + if(dist.build_type && dist.build_type.contains("高层")){ + BasicDBObject udata = new BasicDBObject('is_elevator',true) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata)) + rets << dist + } + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + println c +" "+ rets.size() +// return retMap + } + + def analyElevator(houses, estateType){ + def heights = [] + for(def h : houses){ + if(estateType=="别墅"){ + if(h.height && h.height<=4) { + heights << h.height + } + }else{ + if(h.height && h.height>=10){ + heights << h.height + } + } + } + return heights + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index a04bae008..a660cbe80 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicLong import java.util.regex.Pattern /** - * 小区字段入库(自动入库) + * 小区字段填充入库(自动入库) */ @BatchOwner(Watcher.GaoJuan) class DistrictFieldAddBatch extends BaseBatch{ @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() + def citys = CommonDatautils.findAllCity().reverse() def analysRets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 70249a1f9..db058c5a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -95,7 +95,7 @@ class FangjiaClientUtils { try { if(address){ address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:32080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy new file mode 100644 index 000000000..9ec05ed28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 写字楼字段清洗入库(自动入库) + */ +@BatchOwner(Watcher.GaoJuan) +class OfficeFieldCleanBatch extends BaseBatch{ + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + static main(args) { + runBatch(new OfficeFieldCleanBatch(), args) + } + + @Override + void process(Object args) { + def citys = CommonDatautils.findAllCity().reverse() + for(def c : citys){ + def offices = findAllOffice(c) + strandField(offices) + } + + } + + + def strandField(offices){ + def fields = ["property_fee","build_area","","","","","","","","","",""] + for(def office : offices){ + fields.each{ + def newVal = null + if(it == "property_fee"){ + newVal = FiledStrand.processEstateCharge(office.city,office.get(it), "写字楼") + newVal = newVal.toString() + } + + if(it =="all_area" || it =="build_area"){ + newVal = FiledStrand.processArea(office.get(it)) + if(newVal && newVal > 5000){ + newVal = newVal.toString() + } + } + + if(it =="name"){ + newVal = FiledStrand.processName(office.get(it)) + if(newVal){ + newVal = newVal.toString() + } + } + } + } + + } + + + def findAllOffice(city){ + def sql = "select * from office where city='${city}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + }) + return rets + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy index 945227e7a..7107037b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -15,7 +16,48 @@ class SameDistName { static main(args) { def task = new SameDistName() - task.run() +// task.run() + //地址和地址别名完全一致 +// task.addressPocess() + } + + + + def addressPocess(){ + def citys = CommonDatautils.findAllCity(); + + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"]) + if(dists){ + def index = 0 + for(def dist : dists){ + if(dist.address){ +// if(dist.address.toString().endsWith("弄")){ +// dist.address = dist.address.toString().replace("弄","号") +// } + if(dist.addr_alias && dist.addr_alias.contains(dist.address)){ + def new_address_alias = [] + for(def addr : dist.addr_alias){ + if(!new_address_alias.contains(addr) && addr!=dist.address){ + new_address_alias << addr + } + } + index ++ + + println "[${index}] "+dist.city +" "+dist.name+" 旧:"+dist.addr_alias+" 新:"+ new_address_alias + + def d = new BasicDBObject() + d.put("_id", dist._id) + + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("addr_alias", new_address_alias) + DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) + } + } + } + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index a59b0747a..c26ded24c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -144,5 +144,8 @@ class SeaweedDataMatchProcess { } + public static void main(def agrs){ + doMatch("上海", "静安", "金家巷西弄", null) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy index aa3c3c62c..6b0f2112c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy @@ -15,7 +15,7 @@ String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250& stat() def stat(){ - def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\fjw-提交3.xlsx",0,1) def map=[:] for(def d : datas){ def count = map.get(d['C']) @@ -69,8 +69,8 @@ def findLogs(){ def findFangjiaLogs(){ - def startDate = DateUtil.parseDate("20230329", "yyyyMMdd") - def endDate = DateUtil.parseDate("20230429", "yyyyMMdd") + def startDate = DateUtil.parseDate("20230501", "yyyyMMdd") + def endDate = DateUtil.parseDate("20230531", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def list = [] @@ -84,14 +84,21 @@ def findFangjiaLogs(){ QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false); - def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") +// def q = QueryBuilders.fuzzyQuery("request", "/doAdd2") +// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + + def q = QueryBuilders.fuzzyQuery("request", "/enquire") QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); println houses.size() if (houses) { for (def d : houses) { def reqStr = d.request.toString() + if(!reqStr.contains("/enquire")){ + continue + } def mid = null if (reqStr.contains("mid=")) { @@ -107,5 +114,5 @@ def findFangjiaLogs(){ startDate = nextDate } - ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志") + ExcelUtils.write("D:\\数据\\fjw-提交3.xlsx", data, "查询日志") } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy new file mode 100644 index 000000000..626af0640 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/HouseAnaly.groovy @@ -0,0 +1,232 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils +import org.apache.lucene.queryparser.classic.QueryParser + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class HouseAnaly { + + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new HouseAnaly() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def allRets = [] + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city,"district", ["city","region","name","alias", "estate_type2"]) + def rets = processing(districts) + println "[${city}] 写文件 "+rets.size() + allRets.addAll(rets) + } + ExcelUtils.write("d:\\数据\\样本房源分析(20%).xlsx", allRets, "房源分析") + } + + def processing(dists){ + def rets = new CopyOnWriteArrayList() + rets << ["小区ID","城市","区域","名称","是否是主名","样本量","最大值","最小值","平均值","中位值","10分位置","90分位置","住宅样本量","别墅样本量","商住样本量"] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i=1){ + def ret = analy(dist) + if(ret){ + rets.addAll(ret) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + return rets; + } + + def analy(dist){ + def masterRet = runAnalying(dist._id, dist.city, dist.region, dist.name, "是"); + def alias = dist.alias + for(def aliasName : alias){ + if(!aliasName.contains(dist.name) && !dist.name.contains(aliasName)){ + def aliaRet = runAnalying(dist._id, dist.city, dist.region, aliasName, "否"); + if(aliaRet){ + masterRet.addAll(aliaRet) + } + } + } + if(masterRet.size()>=2){ + def masterAvgPrice = 0 + def masterMidPrice = 0 + def back = false + for(def row : masterRet){ + if(row.get(4)=="是"){ + masterAvgPrice = row.get(8) + masterMidPrice = row.get(9) + }else{ + def pre1 = 0 + def pre2 = 0 + if(masterAvgPrice>0){ + pre1 = java.lang.Math.abs((row.get(8)-masterAvgPrice)*100/masterAvgPrice) + } + if(masterMidPrice>0){ + pre2 = java.lang.Math.abs((row.get(9)-masterMidPrice)*100/masterMidPrice) + } + if(pre1>=20 || pre2>=20){ + back = true + } + } + } + if(back){ + return masterRet + } + + } + return null; + } + + //小区ID 城市 区域 名称 是否是主名 样本量 最大值 最小值 平均值 中位值 10分位置 90分位置 住宅样本量 别墅样本量 商住样本量 + def runAnalying(id, city, region, name, flag){ + def houses = findHouse(city, region, name); + def rets = [] + if(houses && houses.size()>=5){ + def prices = [] + for(def house : houses){ + prices << house.avg_price + } + prices = prices.sort() + def average = (int)(prices.sum() / prices.size()) + def midPrice = getMidPrice(prices) + def price10 = null + def price90 = null + if(houses.size()>=10){ + def postion = Math.round(houses.size()/10) + price10 = prices.get((int)(postion-1)) + price90 = prices.get((int)(houses.size()-postion-1)) + } + def countMap = classType(houses) + +// def pre = java.lang.Math.abs((midPrice-average)*100/average) +// if(pre>=20){ + rets << [id.toString(), city, region, name, flag, prices.size(), prices.get(prices.size()-1), prices.get(0), average, midPrice, price10, price90, countMap.get("district"), countMap.get("villa"), countMap.get("business")] +// } + } + return rets + } + + def classType(houses){ + def district = 0 + def villa = 0 + def business = 0 + for(def h : houses){ + if(h.estate_type){ + if(h.estate_type == "别墅"){ + villa = villa + 1 + continue + }else if(h.estate_type.contains("商住") || h.estate_type.contains("商业") || h.estate_type.contains("商务")){ + business = business + 1 + continue + } + } + def valla = isVilla(h.area, h.name, h.height, h.room) + if(valla){ + villa = villa + 1 + continue + } + district = district + 1 + } + return ["district":district, "villa":villa, "business":business] + } + + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + def getMidPrice(def prices){ + def mid = prices.size()/2 + if(mid > (int)mid){ + return prices.get((int)mid) + }else{ + def total = prices.get((int)(mid-1))+prices.get((int)(mid+1-1)) + return (int)(total/2) + } + return null + } + + + def findHouse(def city, def region, def name){ + def houses = [] + try{ + name = escapeExprSpecialWord(name) + String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= 20230101 and region='${region}' and district_name='${name}'" + ElasticUtils.executeSqlQuery(sql,{ d-> + def r = ["name":d.name, "avg_price":d.avg_price, "area":d.area,"floor": d.floor, "height":d.height,"estate_type": d.estate_type, "build_type":d.build_type] + houses << r +// println r + }) + }catch(Exception e){ + e.printStackTrace() + } + + return houses + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy index ee95b0621..adc26463e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy @@ -21,13 +21,23 @@ class GaoDeMapSearchTest { static main(args) { def task = new GaoDeMapSearchTest(); - def path = "D:\\数据\\光大\\光大测试-V2.xlsx" + def path = "D:\\数据\\富民\\富民测试500.xlsx" // task.structureV2(path) // task.match(path) // task.mapSearch(path) - task.baiduMapSearch(path) - task.checkTrue(path) +// task.baiduMapSearch(path) +// task.checkTrue(path) + task.addFinancialLevel(path) + } + + def addFinancialLevel(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def results = com.fangjia.script.gj.OpenApiUtils.financialRating(null, row['A']) + if(results){ + row['V'] = results.consumptionRating + } + }); } def checkTrue(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 4961abd56..b190a324a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + //广州,佛山 重庆 static main(args) { def task = new BeiKeCrawlByChrome(); @@ -118,6 +119,13 @@ public class BeiKeCrawlByChrome { println city +" 已运行" continue } + def newCity = RegionData.formatCity(city) + if(!newCity){ + if(city=="昆山"){ + newCity = "苏州" + } + println city +" 无法格式化" + } def url = m[1] def newUrl = url+"/chengjiao/" driver.get(newUrl) @@ -130,7 +138,11 @@ public class BeiKeCrawlByChrome { def href = url+ r.attr("href") println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, city, href, regionText) + if(city == "昆山"){ + regionText = "昆山" + } + + def detailUrls = getPageDeail(driver, newCity, href, regionText) println url +" 详情链接:"+detailUrls.size() } } @@ -191,7 +203,10 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e, region) + def dbo = findTrade(durl , "贝壳找房") + if (!dbo) { + parse(city, e, region, driver) + } }else{ stop = true; break; @@ -209,12 +224,20 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e, region){ + def parse(city,e, region, driver){ try { + def detailUrl = e.select(".title a").attr("href") + + driver.get(detailUrl) + def detailDoc = checkDocument(driver, detailUrl); + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() def arrs = title.split(" ") + if(title.contains("车位")){ + detail.type = "车位" + } detail.city = city detail.district_name = arrs[0] detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) @@ -266,28 +289,48 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" - if(detail.total_price && detail.avg_price){ - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } + def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc) + def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc) - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","") + if(liftStr){ + detail.lift = DataFormatUtils.parseLift(liftStr) + detail.lift_household = DataFormatUtils.parseHousehold(liftStr) + } + detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡","")) + + detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","") + detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限","")) + if(detail.total_price && detail.avg_price){ + + detail.url = detailUrl + + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + + detail.floor_info = floorStr + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + def floorInfo = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorInfo && floorInfo.isNumber()) { + detail.height = floorInfo.toInteger() + } } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } }catch(Exception ex){ @@ -392,16 +435,16 @@ public class BeiKeCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def url, source){ def q = new BasicDBObject() // q.put("city", detail.city) // q.put("region", detail.region) // q.put("district_name", detail.district_name) // q.put("area", detail.area) // q.put("total_price", detail.total_price) - q.put("url", detail.url) + q.put("url", url) // q.put("trade_date", detail.trade_date) - q.put("source", detail.source) + q.put("source", source) def dbo = DEAL_PRICE_COLL.findOne(q) return dbo } @@ -448,10 +491,13 @@ public class BeiKeCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(10000) }catch(Exception e){ - Thread.sleep(10000) + Thread.sleep(15000) } return driver } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy new file mode 100644 index 000000000..530579eb8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataFormatUtils.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.paimai + +class DataFormatUtils { + + static def common_used_numerals = ['零':0, '一':1, '二':2, '两':2, '俩': 2, '三':3, '四':4, '五':5, '六':6, '七':7, '八':8, '九':9, '十':10, '百':100, '千':1000, '万':10000, '亿':100000000] + + static def parseLift(def str){ + if(str && str.contains("梯")){ + def lift = str.toString().substring(0, str.toString().indexOf("梯")) + if(lift){ + def val = chineseIntegerPart2Digit(lift) + if(val){ + return (int)val + } + } + } + return null + } + + static def parseHousehold(def str){ + if(str && str.contains("梯") && str.contains("户")){ + def household = str.toString().substring(str.toString().indexOf("梯")+1, str.toString().indexOf("户")) + def val = chineseIntegerPart2Digit(household) + if(val){ + return (int)val + } + } + return null + } + + + static def chineseIntegerPart2Digit(String chineseDigit) + { + Double finalDigit = 0.0; + Integer r = 1; + + for(int i = chineseDigit.length()-1; i>=0;i--) + { + Integer val = common_used_numerals.get(String.valueOf(chineseDigit.charAt(i))); + + if( val >= 10 && i == 0 ) + { + if (val > r) + { + r = val; + finalDigit = finalDigit +val; + } + else + { + r = r * val; + } + } + else if(val >= 10) + { + if (val > r) + { + r = val; + } + else + { + r = r * val; + } + } + else + { + finalDigit = finalDigit + r*val; + } + } + return finalDigit; + } + + static String getValue(String s) { + if (s && s.contains("暂无")) { + return null + } + return s + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index aaf0c9c0d..07241353c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.gj.ccb.FileUtils @@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -196,9 +197,22 @@ public class LianjiaCrawlByChrome { println city +" 已运行" continue } + def newCity = RegionData.formatCity(city) + if(!newCity){ + if(city=="昆山"){ + newCity = "苏州" + } + println city +" 无法格式化" + } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" - driver.get(newUrl) + try{ + driver.get(newUrl) + }catch(Exception e){ + println newUrl +" 链接有问题!" + continue + } + //def detailUrls = getPageUrls(driver, cityUrl) Thread.sleep(10000) def doc = checkDocument(driver, newUrl); @@ -208,9 +222,11 @@ public class LianjiaCrawlByChrome { for(def r : regions){ def regionText = r.text() def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") - + if(city == "昆山"){ + regionText = "昆山" + } println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, city, href, regionText) + def detailUrls = getPageDeail(driver, newCity, href, regionText) println cityUrl +" 详情链接:"+detailUrls.size() } } @@ -282,13 +298,21 @@ public class LianjiaCrawlByChrome { def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") - if(dealDate.getTime()>=endDate.getTime()){ - detailUrls << durl - parse(city, e, regionText) + if(dealDate){ + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + def dbo = findTrade(durl , "链家在线") + if (!dbo) { + parse(city, e, regionText, driver) + } + }else{ + stop = true; + break; + } }else{ - stop = true; - break; + println url +" 成交时间有误!" } + } if(page>=100){ stop = true; @@ -302,85 +326,116 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e, regionText){ + def parse(city,e, regionText, driver){ try { - def title = e.select(".title a").text() - if (title) { - def detail = new BasicDBObject() - def arrs = title.split(" ") - detail.city = city - detail.district_name = arrs[0] - if(regionText){ - detail.region = regionText - }else{ - detail.region = findRegionByThirdPary(detail.city, detail.district_name) - } - if(detail.region){ - detail.region = RegionData.formatRegion(city, detail.region) - } - - def roomStr = arrs[1] - if (roomStr) { - if (roomStr.indexOf("室") > 0) { - def room = roomStr.substring(0, roomStr.indexOf("室")) - if(room && room.isNumber()){ - detail.room = room.toInteger() - } - + def price = e.select(".totalPrice .number").text() + if(price){ + def detailUrl = e.select(".title a").attr("href") + driver.get(detailUrl) + def detailDoc = checkDocument(driver, detailUrl); + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + if(title.contains("车位")){ + detail.type = "车位" } - if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - if(hall && hall.isNumber()){ - detail.hall = hall.toInteger() - } + detail.city = city + detail.district_name = arrs[0] + if(regionText){ + detail.region = regionText + }else{ + detail.region = findRegionByThirdPary(detail.city, detail.district_name) } - } - - if("车位"!=roomStr){ - if(arrs && arrs.size()>=3){ - def areaStr = arrs[2].replace("平米", "") - if (areaStr) { - detail.area = areaStr.toDouble() + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) + } + if(!detail.region){ + def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1) + str = str.substring(0, str.indexOf("',")) + def locArrs = str.split(",") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble()) + if(gmap){ + detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat) } } - def price = e.select(".totalPrice .number").text() - if (price && price.isNumber()) { - detail.total_price = (int)(price.toDouble() * 10000) + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + def room = roomStr.substring(0, roomStr.indexOf("室")) + if(room && room.isNumber()){ + detail.room = room.toInteger() + } + + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hall && hall.isNumber()){ + detail.hall = hall.toInteger() + } + } } - if (detail.total_price && detail.area) { - detail.avg_price = (int) (detail.total_price / detail.area) - } - def tradeStr = e.select(".dealDate").text() - if (tradeStr) { - detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") - } - detail.source = "链家在线" - detail.url = e.select(".title a").attr("href") + if("车位"!=roomStr){ + if(arrs && arrs.size()>=3){ + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + } - if(detail.total_price && detail.avg_price){ - def dbo = findTrade(detail) - if (!dbo) { + + if (price && price.isNumber()) { + detail.total_price = (int)(price.toDouble() * 10000) + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + detail.url = detailUrl + + def detailDiv = JQueryUtils.select(".introContent", detailDoc) + + def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","") + if(liftStr){ + detail.lift = DataFormatUtils.parseLift(liftStr) + detail.lift_household = DataFormatUtils.parseHousehold(liftStr) + } + detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层","")) + + if(detail.total_price && detail.avg_price){ def houseInfo = e.select(".houseInfo").text() def houseInfos = houseInfo.split("\\|") detail.direction = houseInfos[0] detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() + + detail.floor_info = floorStr if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() + def floorStr2 = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr2 && floorStr2.isNumber()) { + detail.height = floorStr2.toInteger() } } - detail.create_date = new Date() detail.m_date = new Date() println detail DEAL_PRICE_COLL.save(detail) } } - } } }catch(Exception ex){ @@ -484,16 +539,16 @@ public class LianjiaCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def url, source){ def q = new BasicDBObject() // q.put("city", detail.city) // q.put("region", detail.region) // q.put("district_name", detail.district_name) // q.put("area", detail.area) // q.put("total_price", detail.total_price) + q.put("url", url) // q.put("trade_date", detail.trade_date) - q.put("url", detail.url) - q.put("source", detail.source) + q.put("source", source) def dbo = DEAL_PRICE_COLL.findOne(q) return dbo } @@ -530,12 +585,14 @@ public class LianjiaCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); +// elemUsername.sendKeys("15021384995"); +// elemPassword.sendKeys("12qwaszx"); - elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("16283166371"); + elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; - Thread.sleep(10000) + Thread.sleep(15000) }catch(Exception e){ Thread.sleep(10000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy new file mode 100644 index 000000000..949e55f51 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/StandardOtherWebRegion.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.gj.region + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 格式化其它网站区域 + */ + + +def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + +DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "region_recognize", mongoDbUrl) + + + +def datas = ExcelUtils.read("D:\\数据\\第三方\\格式化第三方城市.xlsx",0, 1) +for(def d : datas){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("recognizeCity", d['F']) + q.put("recognizeRegion", d['A']) + def count = SERVER_AREA_COLL.count(q) + if(!count){ + def obj = new BasicDBObject() + obj.put("city", d['A']) + obj.put("recognizeRegion", d['A']) + obj.put("recognizeCity", d['F']) + obj.put("createdDate", new Date()) + SERVER_AREA_COLL.save(obj) + }else{ + println d['A']+" "+ d['F'] +" 已存在" + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index ced116b61..4e23f59e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -58,8 +58,9 @@ class CZDataBatch extends BaseBatch{ def commids = findAllJnBankDatas(city) def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); - districtRun(districts, statMap, commids); FzDataUtils.removeNoUpdate(city, districts) + districtRun(districts, statMap, commids); + def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy index bb19fe69d..72df7724e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy @@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ if(dists){ for(def d : dists){ - def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + def distObj = null + if(d.comm_id){ + distObj = CommonDatautils.findDistrictById(d.comm_id) + }else{ + distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + } if(distObj && distObj._id){ - FzDataUtils.converDistrict(distObj, null, null); + if(d.type && d.type==1){ + FzDataUtils.removeById(d.comm_id) + }else{ + FzDataUtils.converDistrict(distObj, null, null); + } println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name updateTaskStatus(d.id, 2); } @@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ def q = new BasicDBObject() q.put("status", 5);//未处理 TASK_COLL.find(q).each {d-> - rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name] + rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id] } return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index bf6e2c177..51e58e28e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -119,6 +119,13 @@ class FzDataUtils { } } + + static def removeById(id){ + def del_data = new BasicDBObject() + del_data.put("num_id", id) + DIST_COLL.remove(del_data) + } + static def saveDistrict(def m){ DBObject q = new BasicDBObject(); q.put("num_id",m.num_id) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index b5f86bf1b..1c287e355 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) +// if(city == "福州"){ + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) +// } + } while(flag){ @@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{ } } } - if(!dbo){ + if(!dbo && isStandardRoad(copyObj.address)){ result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; @@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("city","福州") + qdata.put("city","福州") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 320e65b2f..4fe5dd4a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ } @Override void process(Object args) { - println SHORT_DATE + def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 96709e230..9c6952a70 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -237,7 +237,7 @@ public class DbUtils{ def lng = district.lng / 10000000000.0 sdq.put("sd.point", lat + "," + lng); sdq.put("sd.sfield", "lat_lng"); - sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0)); + sdq.put("sd.distance", String.valueOf((double) 5000 / 1000.0)); query.append('$sd', sdq) def keys = new BasicDBObject()