From 3d525b2e6cd2f6d1144c26db1eb177b44c3462c6 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sun, 24 Apr 2022 11:01:53 +0800 Subject: [PATCH 1/2] =?UTF-8?q?household=5Fcount=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index dc090f531..bd387ed77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -243,7 +243,7 @@ class DistrictIndexBatch extends BaseBatch{ map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger() } if(d.household_count){ - map.household_ount = d.household_count + map.household_count = d.household_count } if(d.lat){ def lMap = [:] From 94d15915b6a70e65d99a2423f162536eac07135b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 6 May 2022 15:28:55 +0800 Subject: [PATCH 2/2] update --- .../ccbevaluation/BigdataBuildingBatch.groovy | 84 ++-- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 18 +- .../script/ccbevaluation/TradeStatLog.groovy | 8 +- .../ccbevaluation/data/CcbDistrictToDB.groovy | 296 +++++++++++- .../chongqing/BlankEvaluationBatch.groovy | 8 +- .../script/chongqing/BlankMatch.groovy | 64 +-- .../chongqing/MatchEvaluateRegionTest.groovy | 239 ++++++++++ .../script/chongqing/MatchEvaluateTest.groovy | 280 +++++++++++ .../script/commData/CommonDatautils.groovy | 18 + .../script/common/DataHandleUtils.groovy | 6 +- .../script/gj/chongqingblank/DataMatch.groovy | 4 +- .../script/gj/crawl/AnjukeDetailCrawl.groovy | 246 ++++++++++ .../script/gj/crawl/CommonCrawl.groovy | 29 ++ .../script/gj/data/TempMatchDistrict.groovy | 2 +- .../gj/district/FangjiaClientUtils.groovy | 17 + .../script/gj/meituan/MatchTest.groovy | 354 ++++++++++++++ .../script/gj/paimai/Caa123Crawl.groovy | 28 +- .../fangjia/script/gj/paimai/DataUtil.groovy | 25 +- .../fangjia/script/gj/paimai/IcbcCrawl.groovy | 348 ++++++++++++++ .../script/gj/paimai/JdPaimaiCrawl.groovy | 9 +- .../script/gj/paimai/TaobaoPaimaiCrawl.groovy | 450 ++++++++++++++++-- .../fangjia/script/loudong/CqfdcMatch.groovy | 128 ++++- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 2 +- .../fangjia/script/util/CommonUtils.groovy | 42 ++ .../com/fangjia/script/util/DbUtils.groovy | 37 ++ .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 27 files changed, 2604 insertions(+), 144 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 42c580735..6ef05438e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{ def q = new BasicDBObject() q.put("city", city) q.put("status", 1) +// q.put("name","辽源一村") def dbcursor = LOU_DONG_STAT.find(q) def map = [:] + def set = new HashSet() for(def db : dbcursor){ + def list = map.get(db.region+"##"+db.name) if(!list){ list = [] map.put(db.region+"##"+db.name, list) } - list << db + if(db.house_num && db.house_num!="--"){ + def key = db.region+"##"+db.name +"##"+db.house_num + if(!set.contains(key)){ + set << key + list << db + } + } } return map } def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){ def rets = [] + def keys = [] distMap.each{k,dist-> def buildings = buildingMap.get(k) if(buildings){ @@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{ def ret = [] def areaData = distCodeMap.get(obj.region) if(areaData){ - ret << cityCodeDatas.get(obj.city) - ret << obj._id.toString() - ret << obj.house_num - ret << "" - ret << "" - ret << areaData.district_code - ret << dist._id.toString() - if(obj.build_year){ - ret << obj.build_year - }else{ + def key = dist._id.toString() +"##"+obj.house_num + if(!keys.contains(key)){ + keys << key + ret << cityCodeDatas.get(obj.city) + ret << obj._id.toString() + ret << obj.house_num + ret << obj.house_num ret << "" - } - def arrs = CcbCodeUtils.parseBuildType(dist.build_type) - if(arrs){ - ret << arrs[0].trim() - }else{ - ret << '' - } + ret << areaData.district_code + ret << dist._id.toString() + if(obj.build_year){ + ret << obj.build_year + }else{ + ret << "" + } + def arrs = CcbCodeUtils.parseBuildType(dist.build_type) + if(arrs){ + ret << arrs[0].trim() + }else{ + ret << '' + } + + ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) + if(obj.floors){ + ret << obj.floors + }else{ + ret << "" + } - ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) - if(obj.floors){ - ret << obj.floors - }else{ ret << "" - } - - ret << "" - ret << "" - ret << "" + ret << "" + ret << "" // if(obj.elevators && obj.elevators>=1){ // ret << "1" // }else{ // ret << "0" // } - ret << "" + ret << "" // if(obj.elevators && obj.elevators>=1) { // ret << obj.elevators//电梯数 // }else{ // ret << "" // } - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << "1" - rets << ret + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << "1" + rets << ret + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 8a8b4a7dc..98fbc2d09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -367,7 +367,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","特房锦绣一方") +// q.put("name","辽源一村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index f75d28656..d7840e5bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,8 +25,8 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); -// task.deleteThirdPartyData() - task.delDealByPe() + task.deleteThirdPartyData() +// task.delDealByPe() } def delDealByPe(){ @@ -370,26 +370,26 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = ["东莞"] //,"杭州""广州", + def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202110 + def month = 202203 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202111) + def rets = findPeGteThirdParty(city, 202204) def total = 0 for(def m : rets){ def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) } - println city +" "+total + println city +" "+rets.size() } } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50" def rets = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance236.eachRow(sql,{r-> def map = [:] map.commid = r.commid map.price = r.target_avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index b0b02bde1..3ed13ebfc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 -// task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -481,8 +481,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2022-02-01" - def endDate = "2022-03-01" + def startDate = "2022-03-01" + def endDate = "2022-04-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 5df3fb0fd..2db54e10a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.DateUtil import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.mongodb.BasicDBObject +import java.util.concurrent.CopyOnWriteArrayList + class CcbDistrictToDB { static main(args) { def task = new CcbDistrictToDB(); // task.run() - task.run1() +// task.run1() +// task.meituanRun() +// task.meituanSqrentRun() +// task.updateDistrict() + task.updateDongguangRegion() } + + def updateDongguangRegion(){ + def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd") + def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"]) + for(def d : dist){ + def t = d.c_date + if(t.getTime()>cDate.getTime()){ + + } + } + } + + def updateDistrict(){ + def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"] + def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd") + def index = 0 + for(def city : citys){ + def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"]) + for(def d : dist){ + def t = d.c_date + if(t.getTime()>cDate.getTime()){ + if(d.name.contains("(")){ + def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1) + def newName = d.name.replace(delstr,"") + index++ + println index+" "+d.city+" "+d.name+" "+newName + def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district") + if(!obj){ + println "update" + def udata = new BasicDBObject() + udata.put("name", newName) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + + d.name = newName + addPrice(d) + + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0) + }else{ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) + } + } + } + } + } + } + + def meituanSqrentRun(){ + ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def city = d['F'] + def region = d['G'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion) { + def name = d['H'] + name = nameFileter(name) + if(name){ + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0) + } + } + },0); + } + + def meituanRun(){ + def index = 0//美团需入库小区合并后.xlsx + def keys = new CopyOnWriteArrayList() + + ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def city = d['F'] + def region = d['G'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['H'] + name = nameFileter(name) + + def formatAddress = null + if(d['E']){ + formatAddress = AddressUtil.formatAddress(city, d['E']) + } + + if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ + def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) + def map = findStandardObject2(city,region, name, d['E'], rets); + if(map && map.address){ + formatAddress = AddressUtil.formatAddress(city, map.address) + if(!formatAddress){ + if(map.address.contains("交叉口")){ + formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) + }else if(map.address.contains("交叉")){ + formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) + } + } + } + } + + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + + + println index+" "+data + + def stop = false + while (!stop){ + if(data._id.toString()){ + addPrice(data) + stop = true; + } + } + + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + },0); + + + + } + + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + def run1(){ def index = 0 - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d -> + + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d -> def city = d['A'] def region = d['B'] def newRegion = RegionData.formatRegion(city, region) @@ -71,6 +289,7 @@ class CcbDistrictToDB { if(name){ def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") if(!dist){ + def data = new BasicDBObject(); data.put("name_cat",0) data.put("status",0) @@ -221,4 +440,77 @@ class CcbDistrictToDB { name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") return name; } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy index 88f5c3bdf..e45fc62c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy @@ -16,7 +16,9 @@ import com.mongodb.DBObject import groovy.sql.Sql import org.apache.lucene.queryparser.classic.QueryParser - +/** + * 重庆估值测试--2小时 + */ class EvaluationBatch { public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; @@ -27,8 +29,8 @@ class EvaluationBatch { static main(args) { def task = new EvaluationBatch() // task.run() - task.run1(); -// task.run2() +// task.run1(); + task.run2() } def run2(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy index 75c731a7f..77b853dad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy @@ -8,42 +8,17 @@ import com.fangjia.script.util.OpenApiClient class BlankMatch { static main(args) { def task = new BlankMatch() -// task.run() + task.run() // task.run1() // task.run2() - task.run3() - } - - def run3(){ - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> - if(!row['C']){ - def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") - if(dist){ - row['C'] = dist._id.toString() - } - } - },0) - } - def run2(){ - def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> - if(!row['C']){ - for(def d : datas){ - if(d['B'] == row['B']){ - row['C'] = d['C'] - row['D'] = d['D'] - row['E'] = d['E'] - row['F'] = d['J'] - break; - } - } - } - },0) +// task.run3() } def run() { //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); - ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + //def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx"; + def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> if(!row['C']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) if(jsonObject != null && jsonObject.precise!=-1) { @@ -54,8 +29,6 @@ class BlankMatch { row['G'] = jsonObject.get("address") } } - - }); } @@ -81,4 +54,31 @@ class BlankMatch { // } }); } + + def run3(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") + if(dist){ + row['C'] = dist._id.toString() + } + } + },0) + } + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + for(def d : datas){ + if(d['B'] == row['B']){ + row['C'] = d['C'] + row['D'] = d['D'] + row['E'] = d['E'] + row['F'] = d['J'] + break; + } + } + } + },0) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy new file mode 100644 index 000000000..bae011454 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -0,0 +1,239 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList + +class MatchEvaluateRegionTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new MatchEvaluateRegionTest() + task.run()//小区匹配 + task.run1()//地图匹配 +// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 + task.addPrice() + task.addAuthor() + } + def run(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['F'] = jsonObject.get("id") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("name") + row['J'] = jsonObject.get("address") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def run1(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['F'] = jsonObject.get("id") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("name") + row['J'] = jsonObject.get("address") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def addPrice(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['L']){ + def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) + if(priceObj){ + row['M'] = priceObj.avg_price + row['N']= (int)(priceObj.avg_price * row['E']) + } + + def avg_price = findAvgPrice(row['M']) + if(row['M'] && avg_price){ + def weekendPrice = row['M'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + + } + + def addAuthor(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def total = 747 + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(index < avg){ + row['P'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + }); + } + + + def addDistrict(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def flag = d['M'] + if(!flag){ + def city = d['E'] + def region = d['F'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['G'] + name = nameFileter(name) + + def formatAddress = null + def address = d['H'] + if(address){ + formatAddress = AddressUtil.formatAddress(city, address) + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + addPrice(data) + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + } + },0) + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name.contains("(")){ + def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1) + name = name.replace(delstr,"") + } + return name; + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy new file mode 100644 index 000000000..e0a41a599 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -0,0 +1,280 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList +import com.fangjia.script.gj.district.FangjiaClientUtils + +class MatchEvaluateTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx" + def total = 1500 + + static main(args) { + def task = new MatchEvaluateTest() +// task.run()//小区匹配 +// task.run1()//地图匹配 +//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 +// task.addPrice() +// task.addArroundPrice() +// task.addAuthor() + + task.stat() + } + + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A'] && !row['K']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['K'] = priceObj.unitPrice + row['L'] = (int) (priceObj.unitPrice * row['C']) + row['O'] = "周边价格" + } + } + }); + } + + + def run(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['D'] && row['B']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['D'] = jsonObject.get("id") + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("address") + row['I'] = jsonObject.get("propertyType") + row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def run1(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['D'] && row['B']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['D'] = jsonObject.get("id") + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("address") + row['I'] = jsonObject.get("propertyType") + row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def addPrice(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['J'] && row['E']){ + def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G']) + if(priceObj){ + row['K'] = priceObj.avg_price + row['L']= (int)(priceObj.avg_price * row['C']) + } + + + def avg_price = findAvgPrice(row['D']) + if(row['K'] && avg_price){ + def weekendPrice = row['K'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + + } + + def addAuthor(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['B']){ + if(index < avg){ + row['N'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + + def addDistrict(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def flag = d['M'] + if(!flag){ + def city = d['E'] + def region = d['F'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['G'] + name = nameFileter(name) + + def formatAddress = null + def address = d['H'] + if(address){ + formatAddress = AddressUtil.formatAddress(city, address) + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + addPrice(data) + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + } + },0) + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name.contains("(")){ + def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1) + name = name.replace(delstr,"") + } + return name; + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } + + def stat(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1) + ExcelUtils.multiThreadingModify(path, { d -> + if(d['B']){ + for(def row : datas){ + if(d['B']==row['B']){ + if(row['K']){ + d['P'] = row['K'] + break + } + } + } + } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 239c9f83c..d6ab63366 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -830,6 +830,7 @@ class CommonDatautils { keys.put("name_cat", 1); keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) + keys.put("sid", 1) DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ @@ -3166,5 +3167,22 @@ class CommonDatautils { } return avg_price; } + + static def findCityAvgRentRadio(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + + def dbcursor = SQMETRE_RENT_COLL.find(q) + def avg_price = null + def radios = [] + for(def d : dbcursor){ + if(d.price_to_rent){ + radios << d.price_to_rent.toInteger() + } + } + def avg_radios = (int)(radios.sum()/radios.size()) + return avg_radios; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy index 7d1ef8018..7fe3204dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy @@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils; class DataHandleUtils { - static main(args){ - - } + static def getStandardRegion(def regionMap, def region){ def newRegion = null; @@ -94,6 +92,8 @@ class DataHandleUtils { BigDecimal deSource = new BigDecimal(dSource); return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue(); } + + static def trimChar(String c) { try { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 9f84f6479..418b6c102 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -25,11 +25,11 @@ class DataMatch { static main(args) { def task = new DataMatch() // task.run1() - task.run2() +// task.run2() // task.run3() // println task.findAddress("昌州街道棠香路") -// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy new file mode 100644 index 000000000..d4f3001b3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy @@ -0,0 +1,246 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeDetailCrawl { + DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") + static main(args) { + def task = new AnjukeDetailCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://chongqing.anjuke.com/community/?from=navigation" + WebDriver driver = new ChromeDriver() + def regions = getRegions(driver, url) + getAllRegions(driver, regions) + + } + + + def getAllRegions(driver, regions){ + for(def url : regions){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def total = getTotal(doc) + if(total) { + if (total > 1250) { + + } else { + def links = getAllLinks(url, driver) + parse(links,driver) + } + } + } + } + + def getAllLinks(url, driver){ + def stop = false + def urls = [] + while(!stop) { + def doc = Jsoup.parse(driver.getPageSource()); + def details = JQueryUtils.select(".list-cell a.li-row", doc) + + for (def d : details) { + urls << d.attr("href") + } + def nextEle = JQueryUtils.select(".next", doc) + if (nextEle) { + driver.findElement(By.cssSelector(".next")).click() + }else{ + stop = true + } + if(urls.size()>20){ + stop = true + } + } + + return urls + } + + def getTotal(doc){ + + def totalText = JQueryUtils.selectText(".total-info", doc) + if(totalText){ + def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim() + if(total && total.isNumber()){ + return total.toInteger() + } + } + return null; + } + + def getRegions(driver, url){ + driver.get(url) + Thread.sleep(3000) + def doc = Jsoup.parse(driver.getPageSource()); +// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) +// if(loginIfm){ +// def href = loginIfm[0].attr("src") +// driver.get(href) +// driver.findElementById("pwdTab").click() +// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框 +// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框 +// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。 +// elemUsername.sendKeys("17740818975"); +// elemPassword.sendKeys("12qwaszx"); +// btn.submit(); // 点击行为; +// +// def regionEles = JQueryUtils.select(".region-parents li a", doc) +// def urls = [] +// for(def e : regionEles){ +// urls << e.attr("href") +// } +// return urls; +// } + def regionEles = JQueryUtils.select(".region-parents li a", doc) + def urls = [] + for(def e : regionEles){ + urls << e.attr("href") + } + return urls; + } + + def parse(def links, driver){ + def rets = [] + for(def link : links){ + def district = parsing(link, driver) + if(district){ + rets << district + + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + rets = [] + } + } + } + if (rets.size()> 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + } + } + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = detailColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + detailColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + detailColl.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + return add + } + + + + def parsing(url, driver){ + + def district = [:] + def stop = false + def count = 0 + while(!stop){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def headers = JQueryUtils.select(".crumbs a", doc) + if(headers){ + if (headers && headers.size() >= 3) { + district.city = headers[1].text().replace("小区", "") + district.region = headers[2].text().replace("小区", "") + district.name = headers[headers.size() - 1].text().replace("二手房", "") + } + + + district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc) + + def priceStr = JQueryUtils.selectText(".average", doc) + + if (priceStr && priceStr.isNumber()) { + district.avg_price = priceStr.toInteger() + } + + def infoDiv = JQueryUtils.select(".info-list", doc) + district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) + district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) + district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) + district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) + + def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() + if (houseCountStr) { + houseCountStr = houseCountStr.replace("户", "") + if (houseCountStr) { + district.house_count = houseCountStr.toInteger() + } + } + + district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) + district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) + def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) + if(carSiteStr){ + if(carSiteStr.contains("(")){ + carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("(")) + } + if(carSiteStr && carSiteStr.isNumber()){ + district.car_site = carSiteStr + } + } + district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) + district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) + district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) + district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) + district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.url = url + district.c_date = new Date() + district.m_date = new Date() + district.root = "anjuke.com" + + stop = true; + }else{ + Thread.sleep(3000) + count++ + } + if(count>10){ + stop = true; + } + } + + return district + } + + String getValue(String s){ + if(s && s.contains("暂无")){ + return null + } + return s + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy new file mode 100644 index 000000000..f19c4cd3c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.gj.crawl + +import org.jsoup.Jsoup + +class CommonCrawl { + static def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy index a36cf12a7..13b8fdde9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -17,7 +17,7 @@ class TempMatchDistrict { } def stat(){ - def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"] + def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] def total = 0 for(def city : citys){ def count = CommonDatautils.findAllDistrictCount("district", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 44f6d4a9b..12d2f77b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -83,6 +83,23 @@ class FangjiaClientUtils { return jsonObj; } + public static def arroundPrice(city,address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100" +// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static Map JsonToMap(JSONObject json) { Map columnValMap = new HashMap(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy new file mode 100644 index 000000000..6b4872d19 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy @@ -0,0 +1,354 @@ +package com.fangjia.script.gj.meituan + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId + +class MatchTest { + static main(args) { + def task = new MatchTest() +// task.run() +// task.addNum() + task.dongguang() +// task.meituanDel() + } + + def meituanDel(){ + def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" + def index = 0 + ExcelUtils.modify(path, { row -> + def flag = row['O'] + if(flag){ + def city = row['F'] + def region = row['G'] + def name = nameFileter(row['H']) + def obj = null + if(city == "东莞"){ + obj = CommonDatautils.findDistrict(city, null, name, "district") + }else{ + obj = CommonDatautils.findDistrict(city, region, name, "district") + } + if(obj){ + index++ + println index +" "+ city +' '+ region+" "+name + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(obj._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + },0); + } + + def dongguang(){ + def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx + ExcelUtils.modify(path, { row -> + def region = null + if(row['B'] && row['B']=="NULL" ){ + region = null + } + def name = nameFileter(row['H']) + def address = row['E'] + if (name && name.contains("湖畔新家")) { + + def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district") + if(obj){ + def json = GaoDeApiUtils.geo(row['F'], address) + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + def twinshop = getStreetInfo(m.location) + if(twinshop){ + def newRegion = RegionData.formatRegion("东莞",twinshop) + if(newRegion){ + def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district") + if(dbObj){ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id)) + }else{ + println "update" + def udata = new BasicDBObject() + udata.put("region", newRegion) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + } + // println address +" "+twinshop + } + } + } + } + },0) + + } + + def getStreetInfo(def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + } + } + } + return township; + } + + def addNum(){ + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx + def index = 0 + ExcelUtils.modify(path, { row -> + index++ + row['A'] = index + },0) + } + + def run() { + def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + def radioMap = [:] + for(def city : citys){ + +// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); +// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) +// radioMap.put(city, avg_radio) + + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + + } + + + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + def region = null + if(row['C'] && row['C']=="NULL" ){ + region = null + } + def name = row['D'] + def address = row['F'] + def city = RegionData.formatCity(row['B']) + + if(name && !row['N']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } + if(rMap.level){ + row['P'] = rMap.level + } + } + } + } + + if (name && row['N']) { +//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name) +// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null) +// if (jsonObject != null && jsonObject.precise != -1) { +// row['F'] = jsonObject.get("city") +// row['G'] = jsonObject.get("district") +// row['H'] = jsonObject.get("name") +// row['I'] = jsonObject.get("propertyType") +// row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") +// } + + if(row['G'] && row['H'] && row['I']){ +// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H']) +// if(priceObj){ +// row['M'] = priceObj.avg_price +// } +// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H']) +// if(sqmetreRentPrice){ +// row['N'] = sqmetreRentPrice +// } + + if(row['N'] && !row['O']){ + def avg_rent_radio = radioMap.get(city) + row['O'] = (int)(row['N']/avg_rent_radio) + } + } + + } + }) + } + + def getArroundDistrictPriceByAddress(def city, def address, def levels){ + def json = GaoDeApiUtils.geo(city, address) + def location = null + def detail = new BasicDBObject() + def level = null + def rank = null + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + location = m.location + level = m.level + } + } + if(location){ + def arrs = location.split(",") + detail.lng = arrs[0].toString().toDouble()*10000000000L + detail.lat = arrs[1].toString().toDouble()*10000000000L + } + if(detail.lng){ + detail.city = city + + List list = DbUtils.findAroundDistrictByRadius(detail, 3000) + if(list && list.size()>10){ + list = sortByDistance(detail, list); + list = list[0..9] + } + def arroundAvgPrice = getArroundAvgPrice(list) + if(arroundAvgPrice){ + + def mod = Integer.MAX_VALUE; + for(def d : levels){ + def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice) + if(newMod o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + return datas; + } + + def getArroundAvgPrice(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + if(prices){ + def avg_price = (prices.sum()/prices.size()).toInteger() + return avg_price; + } + return null + } + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + return name; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index afaae338f..fb358e503 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -42,7 +42,6 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() def proviceMap = getDocument(driver); def citys = getCity(driver,proviceMap) @@ -59,9 +58,11 @@ def getDetailLinks(driver, cityList){ citys.each{city,id-> def urls = [] if(!zxcitys.contains(cityMap.provice)){ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) }else{ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) } def doc = Jsoup.parse(driver.getPageSource()); @@ -89,7 +90,7 @@ def getDetailLinks(driver, cityList){ } } - def rets = batchParse(urls); + def rets = batchParse(urls, driver); saveToDB(rets); println city +" "+urls.size() } @@ -124,7 +125,7 @@ def getCity(driver, proviceMap){ for(def c : cityEles){ def cityid = c.attr("data-cityid") def city = c.text() - if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ + if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ cityMap.put(city, cityid) } @@ -149,7 +150,7 @@ def getCity(driver, proviceMap){ def getDocument(driver){ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport="); WebDriverWait wait = new WebDriverWait(driver,20) wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); def doc = Jsoup.parse(driver.getPageSource()); @@ -159,7 +160,7 @@ def getDocument(driver){ for(def ele : proviceEles){ def provice = ele.text() def value = ele.attr("data-proviceid") - if(value && provice!="北京市" && provice!="天津市"){ + if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){ provinceMap.put(provice, value) } } @@ -167,8 +168,8 @@ def getDocument(driver){ return provinceMap; } -def batchParse(def urls){ - WebDriver driver = new ChromeDriver() +def batchParse(def urls, driver){ + def ret = [] for(def map :urls){ def doc = getDocument(map.url, driver) @@ -280,7 +281,7 @@ def parse(province, city, url,doc){ result.lookCount = JQueryUtils.selectText(".look_count span", doc) def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc) - if(startDateStr){ + if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){ startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() } @@ -329,6 +330,13 @@ def parse(province, city, url,doc){ result.tradeAvgPrice = (int) (result.tradePrice / result.area) } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + result.content = DataUtil.zip(doc.toString()) result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() result.url = url diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy index 8cce171c6..7e81c3afd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -16,6 +16,27 @@ import java.util.regex.Matcher import java.util.regex.Pattern class DataUtil { + + + static def checkMinDate(curDate, checkDate){ + if(curDate){ + def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd") + if(curDate.getTime()maxDate.getTime()){ + return null + } + } + return curDate + } static def getAreaNumber(def input, def maxArea) { Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)') Matcher matcher = pattern.matcher(input) @@ -112,10 +133,10 @@ class DataUtil { if (list.size() >= 1) { String s1 = list.get(0) if (s1.contains("万元")) { - s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "") + s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","") return (int) (s1.toDouble() * 10000) } else { - s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "") + s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","") if (s1.isInteger()) { return s1.toInteger() } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy new file mode 100644 index 000000000..b432108a2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy @@ -0,0 +1,348 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import javafx.scene.input.DataFormat +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class IcbcCrawl { + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new IcbcCrawl(); + task.run(); + } + + def run(){ + def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + driver.get(url) + driver.findElementById("category_SM310000000000000006").click() + Thread.sleep(3000) + + + def urls = getAllDetailLinks(driver) + println "共:"+urls.size() + + detailCrawling(urls, driver) + + } + + def getAllDetailLinks(driver){ + def stop = false + def urls = [] + while (!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc) + if(eles){ + for(def e : eles){ + def href = e.select(".li_2_name a").attr("href") + urls << "https://gf.trade.icbc.com.cn"+href + } + def nextPage = JQueryUtils.select("a:contains(下一页)", doc) + if(nextPage){ + driver.findElement(By.linkText("下一页")).click(); + }else{ + stop = true + } + }else{ + stop = true + } +// if(urls.size()>=10){ +// stop = true +// } + } + return urls; + } + + def detailCrawling(def urls, driver){ + def rets = [] + for(def url : urls){ + driver.get(url) + Thread.sleep(5000) + def doc = Jsoup.parse(driver.getPageSource()); + def result = parser(doc, url) + if (result) { + rets << result + println rets.size() + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + } + + def parser(doc, url){ + def result = [:] + if(doc){ + def title = JQueryUtils.selectText("#prodnameb", doc) + def id = url.substring(url.indexOf("="), url.length()) + if(title){ + + def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc) + if(cityStr && cityStr.contains("...")){ + def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc) + if(cityEles){ + cityStr = cityEles[0].attr("title") + } + } + if(cityStr && cityStr.contains("省")){ + result.province = cityStr.substring(0, cityStr.indexOf("省")) + result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length()) + }else{ + if(cityStr.contains("自治区")){ + result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3) + result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length()) + }else{ + result.city = cityStr.substring(0, cityStr.indexOf("市")) + } + } + + if(!result.city){ + return null + } + result.title = title + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + + result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","") + + def t1 = JQueryUtils.selectText("#pricep", doc) + if(t1 && t1.contains("原因")){ + result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length()) + } + result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc) + def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#visitorCount"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#remindPPNum"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim() + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc) + if(startPriceEles) { + def startPriceStr = startPriceEles[0].attr("title") + if(startPriceStr){ + startPriceStr = startPriceStr.replace(",", "").replace("¥","") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + } + + } + + + def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(addPriceEles){ + def addPriceStr = addPriceEles[0].attr("title") + if(addPriceStr){ + addPriceStr = addPriceStr.replace(",", "").replace("¥","") + if(addPriceStr) { + result.addPrice = addPriceStr.toDouble().toInteger() + } + } + } + + def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc) + if(evaluationPriceEles){ + def evaluationPriceStr = evaluationPriceEles[0].attr("title") + if(evaluationPriceStr){ + evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","") + if(evaluationPriceStr.isInteger()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(evaluationPriceStr.toDouble()) + } + } + } + + def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(depositPriceEles){ + def v = depositPriceEles[0].attr("title") + if(v){ + v = v.replace(",", "").replace("¥","") + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + } + } + + result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc) + result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc) + result.court = JQueryUtils.selectText("#goToCourt", doc) + def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc) + if(mobileEles){ + result.mobile = mobileEles[0].attr("title") + } + + def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + + def trs = JQueryUtils.select([".rygtable table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + if(d_tds[2].text() && d_tds[2].text()!="/"){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + + }else { + if(d_tds[1].text() && d_tds[1].text()!="/") { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs =JQueryUtils.select("#scroll li img", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("src") + if(href.contains("icbc.com")) { + imgLinks << "https:"+href + } + } + result.imgUrls = imgLinks + } + + + def detailStr = JQueryUtils.selectText(".detail_list", doc) + if(detailStr){ + def area = DataUtil.getAreaNumber(detailStr, 5000) + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = detailStr.toString() + result.url = url + result.mDate = new Date() + result.source = "融e够" + return result; + } + + } + return null + } + + def getCallCount(id){ + def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id + def s = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(s){ + JSONObject jsonObject = JSONObject.fromObject(s); + println jsonObject + } + } + + + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy index aac9c7315..3ea9f1d4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -39,8 +39,6 @@ class JdPaimaiCrawl { println "共:"+ids.size() parserList( existsIds,ids, driver) - - } def getList(driver){ @@ -478,6 +476,13 @@ class JdPaimaiCrawl { } } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString() result.url = url diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy index f0b328f11..68b73d58e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy @@ -1,7 +1,12 @@ package com.fangjia.script.gj.paimai import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.DateUtils +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver class TaobaoPaimaiCrawl { static main(args) { @@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl { task.run(); } def run(){ - def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1" - def regionUrls = regionCrawl(seedUrl); - def pageUrls = pageUrlCrawl(regionUrls) - detailCrawl(pageUrls) + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); +// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1" + def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1" + WebDriver driver = new ChromeDriver() + getAllCityRegions(driver, url); } - def regionCrawl(def seedUrl){ - def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk"); - def regionUrls = [] - if(document){ - def eles = JQueryUtils.select(".small-subcondion a", document) - if(eles){ - for(def el : eles){ - def reiongUrl = el.attr("href").trim() - if(reiongUrl.startsWith("//")){ - regionUrls << "http:"+reiongUrl + def getAllCityRegions(driver, url){ + + def provinces = getAllProvince(driver, url) + def citys = ["天津","上海","北京","重庆"] + + for(def province : provinces){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles= JQueryUtils.select(".unfold .J_FilterCity a") + for(def cityEle : cityEles){ + def city = cityEle.text() + if(citys.contains(city)){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + doc = Jsoup.parse(driver.getPageSource()); + + driver.findElement(By.cssSelector(".J_FilterCity a")).click() + doc = Jsoup.parse(driver.getPageSource()); + def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a") + for(def r : regionEles){ + def m = [:] + m.province = province + m.city = city + m.region = r.text() + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } + }else{ + def m = [:] + m.province = province + m.city = city + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } } } - return regionUrls; } - def pageUrlCrawl(regionUrls){ + def crawlLinks(def m, def driver){ + def stop = false def urls = [] - for(def url : regionUrls){ - def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk"); - if(document){ - def pages = JQueryUtils.selectText(".page-total",document) - if(pages){ - pages = pages.toInteger() - for(def page=1;page -1) { + result.address = addr.substring(0, addr.indexOf("室") + 1) + } else { + result.address = addr + } + + if (result.city) { + if (result.address.contains(city + "市")) { + result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1) + } else { + result.address = result.address.substring(result.address.indexOf(city) + city.length()) + } + } + + + if (result.address.indexOf("】") > -1) { + result.address = result.address.substring(result.address.indexOf("】") + 1) + } + + + if (result.city) { + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + } + if (statusDesc) { + def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1) + result.status = DataUtil.getPaiMaiType(status) + result.statusDesc = statusDesc + } + + def statusEle = JQueryUtils.select(".status-tip", doc) + if (statusEle) { + def sEle = JQueryUtils.select(".status-ing", doc) + if (sEle) { + result.status = "进行中" + } else { + result.status = "即将开始" + } + } + def curPrice = JQueryUtils.selectText(["#sf-price"], doc) + if (curPrice) { + if (curPrice.contains("当前价")) { + curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (curPrice) { + curPrice = curPrice.replace(",", "") + if (curPrice.isInteger()) { + result.curPrice = curPrice.toInteger() + } else { + result.curPrice = (int) (curPrice.toDouble()) + } + } + } else if (curPrice.contains("成交价")) { + def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (tradePrice) { + tradePrice = tradePrice.replace(",", "") + if (tradePrice.isInteger()) { + result.tradePrice = tradePrice.toInteger() + } else { + result.tradePrice = (int) (tradePrice.toDouble()) + } + + } + } else if (curPrice.contains("起拍价")) { + def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (startPrice) { + startPrice = startPrice.replace(",", "") + if (startPrice.isInteger()) { + result.startPrice = startPrice.toInteger() + } else { + result.startPrice = (int) (startPrice.toDouble()) + } + + } + } + } + + def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc) + if (finishStr) { + if (finishStr.contains("结束时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } else if (finishStr.contains("成交时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } + } + + def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc) + if (enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc) + if (callCount) { + result.callCount = callCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#J_Looker"], doc) + if (lookCount) { + result.lookCount = lookCount.toInteger() + } + def tds = JQueryUtils.select(["#J_HoverShow td"], doc) + if (tds && tds.size() > 0) { + for (def td : tds) { + def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "") + if (str) { + if (str.contains("变卖价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffPrice = p.toInteger() + } else { + result.sellOffPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("加价幅度")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.addPrice = p.toInteger() + } else { + result.addPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("类型")) { + + result.type = td.select(".pay-type").text() + + } else if (str.contains("保证金")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.depositPrice = p.toInteger() + } else { + result.depositPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("变卖周期")) { + + result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("竞价周期")) { + + result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("优先购买权")) { + + result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("变卖预缴款")) { + def str1 = td.select("span:eq(1) .J_Price").text() + def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffAdvancePay = p.toInteger() + } else { + result.sellOffAdvancePay = (int) (p.toDouble()) + } + } + + } else if (str.contains("评估价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.evaluationPrice = p.toInteger() + } else { + result.evaluationPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("延时周期")) { + + result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("起拍价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.startPrice = p.toInteger() + } else { + result.startPrice = (int) (p.toDouble()) + } + } + } + } + } + } + + result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc) + + def trs = JQueryUtils.select(["#J_desc table tr"], doc) + if (trs) { + def detailMap = [:] + for (def tr : trs) { + def d_tds = tr.select("td") + if (d_tds && d_tds.size() >= 2) { + if (d_tds.size() == 3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } else { + if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + + if (detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + + if (!result.address) { + result.address = detailMap.get("拍品名称") + if (!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs = JQueryUtils.select([".slide-bigpic img"], doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + + } + result.imgUrls = imgLinks + } + } + + //面积 + def article = JQueryUtils.selectText(["#J_ItemNotice"], doc) + if (article) { + def area = DataUtil.getAreaNumber(article, 5000) + if (!area) { + article = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (!area) { + article = JQueryUtils.selectText(["body"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (area) { + result.area = area.toDouble() + if (result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble()) } } } - } - return urls; - } - def detailCrawl(pageUrls){ - if(pageUrls){ - for(def pageUrl : pageUrls){ - def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk"); - if(document){ - def pages = JQueryUtils.select(".page-total", document) + + def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (s1) { + def dateRangs = DataUtil.getDateRange(s1) + if (dateRangs) { + for (def j = 0; j < dateRangs.size(); j++) { + def date = dateRangs[j] + def pattern = null + if (date.endsWith("日时") || date.endsWith("日")) { + pattern = "yyyy年MM月dd日" + } else { + + def ind = date.indexOf("日") + if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) { + date = date + "时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if (j == 0) { + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + } else { + if (!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + + if (!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) } } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + def estateType = DataUtil.findEstateType(s) + if (estateType) { + result.matchEstateType = estateType + } + + result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() + result.url = url + result.mDate = new Date() + results << result } + return results; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index 1ce1c48c7..bd59e65c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -1,20 +1,38 @@ package com.fangjia.script.loudong +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong class CqfdcMatch { + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + static main(args) { def task = new CqfdcMatch(); // task.run() // task.stat() - task.export2() +// task.export2() +// task.matchLoudong() + + task.addLoudong() } def export(){ def q = new BasicDBObject() @@ -139,6 +157,29 @@ class CqfdcMatch { return rets; } + def findAllLoundongByHouseNum(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000) + def rets = [] + def map = [:] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num + if(d.unit_num){ + key = key +"##"+d.unit_num + } + def detailMap= map.get(key) + if(!detailMap){ + detailMap = [:] + map.put(key,detailMap) + } + + if(detailMap.height){ + + } + } + return rets; + } + def stat(){ def q = new BasicDBObject() @@ -168,6 +209,91 @@ class CqfdcMatch { } + def matchLoudong(){ + def names = findAllLoudong("重庆") + print names.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(names.size()) + + def retMap = new ConcurrentHashMap() + for(def i=0;i + def arrs = key.split("#") + + def q = new BasicDBObject("city", arrs[0]) + q.put('region', arrs[1]); + q.put('name', arrs[2]); + + def udata = new BasicDBObject() + udata.put("district_id", m.id.toString()) + udata.put("name_in_seaweed", m.name) + udata.put("district_no", m.sid) + HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true); + index++ + println index + } + } + + def findAllLoudong(city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district_id",new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def names = new CopyOnWriteArrayList() + for(def db : dbcursor){ + def key = db.city +"#"+db.region+"#"+db.name + if(!names.contains(key)){ + names << key + } + } + return names; + } + + def addLoudong(){ + + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ace3084c1..ee0977a28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -192,7 +192,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] + def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index d37680fac..fe433f087 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -245,6 +245,48 @@ public class CommonUtils { } + + static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) { + + + calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate) + if (estateType && estateType.contains("别墅")) { + estateType = "别墅" + } else if (estateType) { + estateType = "小区" + } + + DBObject cmd = new BasicDBObject() + cmd.put("city", city) + cmd.put("region", region) + cmd.put("name", name) + cmd.put('estate_type', estateType) + cmd.put("calc_date", calcDate) + cmd.put("calc_type", 0) + println(cmd) + DBObject data = new BasicDBObject() + data.put("category", "square_rent") + data.put("priority", 3) + data.put("status", 0) + data.put("remark", 'week') + data.put("c_date", new Date()) + + + if(isTest){ + data.put("ip", "192.168.0.231") + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + }else{ + data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)]) + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + } + if(sync){ + while (COMMAND_COLL.count(cmd) != 0){ + Thread.sleep(1000) + } + } + + } + /** * 重算小区均价 * @param city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 3b83880cd..69a75d454 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -314,6 +314,43 @@ public class DbUtils{ } return districts + } + + /** + * 查询附近一共里内小区的数量 + */ + def static findAroundDistrictByRadius(BasicDBObject district, def radius){ + def districts = [] + if(hasGeo(district)){ + def query = new BasicDBObject() + query.append("cat", "district") + query.append("city", district.city) + query.append("visible", 0) + DBObject sdq = new BasicDBObject(); + def lat = district.lat / 10000000000.0 + def lng = district.lng / 10000000000.0 + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radius / 1000.0)); + query.append('$sd', sdq) + + def keys = new BasicDBObject() + keys.append("city", 1) + keys.append("region", 1) + keys.append("name", 1) + keys.append("lat", 1) + keys.append("lng", 1) + keys.append("lat2", 1) + keys.append("lng2", 1) + keys.append("estate_type2", 1) + DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo-> + if(!district._id || dbo._id != district._id){ + districts << dbo + } + } + } + return districts + } /** * 查询小区均价 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 3810aa9f3..b7fe9aa99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220501", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0