From 83c0f3bfffc5db0fcfe7b8f8f71a2487eb3792ff Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 18 May 2022 18:35:40 +0800 Subject: [PATCH 01/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../OracleDistrict2Mongo85.groovy | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy new file mode 100644 index 000000000..b143277b9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.lucene.queryparser.classic.QueryParser +import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * oralce中小区统计数据同步至85中 + */ +class OracleDistrict2Mongo85 { + + static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017) + + static main(args) { + // 只写入重庆市数据 + def sql = "select distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ WHERE CITYID='500000'" + def count = 0 + sqlInstance.eachRow(sql, { r -> + def region = RegionData.formatRegion("重庆市", r['distname']) + DBObject dbObject = new BasicDBObject("_id", r['commid']) + DBObject updateObject = new BasicDBObject("city", "重庆").append("region", region) + .append("name", r['commname']) + .append('subway', Integer.parseInt(r['subway'].toString())) + .append("traffic", Integer.parseInt(r['traffic'].toString())) + .append("bank", Integer.parseInt(r['bank'].toString())) + .append("primary", Integer.parseInt(r['primary'].toString())) + .append("junior", Integer.parseInt(r['junior'].toString())) + .append("hospital", Integer.parseInt(r['hospital'].toString())) + .append("funeral", Integer.parseInt(r['funeral'].toString())) + .append("c_date", new Date()); + communityPoiConn.update(dbObject, updateObject, true, false) + count++; + if (count % 500 == 0) { + println "已写入" + count + "条小区数据" + } + }); + } +} From b68dca74dbb778e45c66298901b7079f838d416d Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 18 May 2022 20:19:14 +0800 Subject: [PATCH 02/12] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A4=9A=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/OracleDistrict2Mongo85.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy index b143277b9..0855b1a8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData +import com.fangjia.core.util.DistrictCodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils @@ -40,12 +41,13 @@ class OracleDistrict2Mongo85 { static main(args) { // 只写入重庆市数据 - def sql = "select distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ WHERE CITYID='500000'" + def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ" def count = 0 sqlInstance.eachRow(sql, { r -> - def region = RegionData.formatRegion("重庆市", r['distname']) + def city = DistrictCodeUtils.getCityName(r['cityid']) + def region = RegionData.formatRegion(city, r['distname']) DBObject dbObject = new BasicDBObject("_id", r['commid']) - DBObject updateObject = new BasicDBObject("city", "重庆").append("region", region) + DBObject updateObject = new BasicDBObject("city", city).append("region", region) .append("name", r['commname']) .append('subway', Integer.parseInt(r['subway'].toString())) .append("traffic", Integer.parseInt(r['traffic'].toString())) From f2d596eeb39730fd09df6e3a509685b625007761 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 20 May 2022 10:53:38 +0800 Subject: [PATCH 03/12] update --- .../data/CcbDistNameMatch.groovy | 117 +++ .../chongqing/MatchEvaluateRegionTest.groovy | 27 +- .../script/chongqing/MatchEvaluateTest.groovy | 18 +- .../fangjia/script/chongqing/PocStat.groovy | 763 ++++++++++++++++++ .../fangjia/script/chongqing/PocTest.groovy | 223 +++-- .../script/commData/CommonDatautils.groovy | 33 +- .../fangjia/script/gj/EsHousePriceStat.groovy | 20 +- .../script/gj/crawl/AnjukeAppCrawl.groovy | 198 +++++ .../script/gj/crawl/AnjukeDetailCrawl.groovy | 234 ++++-- .../gj/crawl/AnjukeDistrictMatch.groovy | 89 ++ .../fangjia/script/gj/crawl/AnjukeToDB.groovy | 628 ++++++++++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 8 +- .../script/gj/paimai/JdPaimaiCrawl.groovy | 157 ++-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 34 +- .../script/gj/paimai/PaimaiTest.groovy | 19 +- .../fangjia/script/loudong/CqfdcMatch.groovy | 35 +- .../script/thirdPriceCrawl/DataStat.groovy | 6 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 7 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- .../fangjia/script/util/OpenApiClient.groovy | 21 +- 20 files changed, 2368 insertions(+), 273 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy new file mode 100644 index 000000000..32725d092 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy @@ -0,0 +1,117 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +class CcbDistNameMatch { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new CcbDistNameMatch(); +// task.run() + task.run1() + } + + def run1(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + if(d['D'].contains("、")){ + println d['D'] + } + def arrs = d['D'].split("、") + def list = [] + def flags = [] + for(def s : arrs){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district") + if(dist){ + list << dist.name + flags << "完全一致" + }else{ + def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion) + if(jsonObject != null && jsonObject.precise!=-1) { + if(jsonObject.get("district") == newRegion){ + list << jsonObject.get("name") + flags << "匹配接口" + } + } + } + + } + if(list){ + list = list.unique() + d['E'] = list.join(",") + d['F'] = flags.join(",") + } + } + } + }) + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d -> + def regionCode = null + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if(dist){ + d['D'] = dist.name + def count = findDistrictByOracle(dist._id.toString()) + if(count && count>0){ + d['F'] = 1 + }else{ + d['H'] = dist.estate_type2 + d['J'] = dist.name_cat + } + }else{ + def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district") + if (rets) { + if(rets.size()>1){ + d['E'] = "别名-多个" + }else{ + dist = rets[0] + d['E'] = d['C'] +"--"+dist.name + def count = findAliasByOracle(dist._id.toString()) + if(count && count>0) { + d['G'] = 1 + }else{ + d['H'] = dist.estate_type2 + d['J'] = dist.name_cat + } + } + } + } + } + } + }); + } + + def findDistrictByOracle(commid){ + def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } + + def findAliasByOracle(commid){ + def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy index cd0df4d4c..b35f27015 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -23,22 +23,22 @@ class MatchEvaluateRegionTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx" def total = 1500 static main(args) { def task = new MatchEvaluateRegionTest() - task.run()//小区匹配 - task.run1()//地图匹配 +// task.run()//小区匹配 +// task.run1()//地图匹配 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 task.addPrice() - task.addArroundPrice() - task.addAuthor() +// task.addArroundPrice() +// task.addAuthor() } def run(){ ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -57,7 +57,7 @@ class MatchEvaluateRegionTest { def run1(){ ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -82,7 +82,7 @@ class MatchEvaluateRegionTest { row['N']= (int)(priceObj.avg_price * row['E']) } - def avg_price = findAvgPrice(row['M']) + def avg_price = findAvgPrice(row['F']) if(row['M'] && avg_price){ def weekendPrice = row['M'].toString().toDouble() def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) @@ -95,12 +95,12 @@ class MatchEvaluateRegionTest { def addArroundPrice(){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['A'] && !row['K']) { - def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if(!row['M']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D']) if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { def priceObj = arroundObj.result row['M'] = priceObj.unitPrice - row['N'] = (int) (priceObj.unitPrice * row['C']) + row['N'] = (int) (priceObj.unitPrice * row['E']) row['Q'] = "周边价格" } } @@ -109,21 +109,22 @@ class MatchEvaluateRegionTest { def addAuthor(){ def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] - def total = 747 def personTotal = authors.size() def avg = Math.ceil(total/personTotal) def index = 0 def count = 0; ExcelUtils.modify(path, { row -> + if(row['B']){ if(index < avg){ - row['P'] = authors[count] + row['N'] = authors[count] } index++ if(index >= avg){ count++ index = 0 } + } }); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy index 936b3b582..30997779c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -28,14 +28,14 @@ class MatchEvaluateTest { static main(args) { def task = new MatchEvaluateTest() -// task.run()//小区匹配 -// task.run1()//地图匹配 -//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 -// task.addPrice() -// task.addArroundPrice() -// task.addAuthor() + task.run()//小区匹配 + task.run1()//地图匹配 +// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 + task.addPrice() + task.addArroundPrice() + task.addAuthor() - task.stat() +// task.stat() } def addArroundPrice(){ @@ -57,7 +57,7 @@ class MatchEvaluateTest { // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['D'] && row['B']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false) if(jsonObject != null && jsonObject.precise!=-1) { row['D'] = jsonObject.get("id") row['E'] = jsonObject.get("city") @@ -77,7 +77,7 @@ class MatchEvaluateTest { // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['D'] && row['B']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true) if(jsonObject != null && jsonObject.precise!=-1) { row['D'] = jsonObject.get("id") row['E'] = jsonObject.get("city") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy new file mode 100644 index 000000000..8a4c43d95 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy @@ -0,0 +1,763 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PocStat { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + + static main(args) { + def task = new PocStat(); + task.regionDistStat(); +// task.sampleStat() +// task.tradeStat() +// task.paimaiStat() +// task.evalStat() +// +// task.fileldStat() + } + + def regionDistStat(){ + def city = "重庆" + def regionMap = CommonDatautils.getRegionsByCity(city) + def set =regionMap.keySet(); + def rows = [] + rows << ["区域","小区量"] + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"]) + for(def region : set){ + def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district") + + def regionCount = 0 + def regionHouseLDCount = 0 + for(def d : dists){ + if(d.region == region){ + def loudongCount = CommonDatautils.findLoudong(d._id.toString()) + if(loudongCount && loudongCount>0){ + regionCount = regionCount+ loudongCount + } + + def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString()) + if(houseLouDongCount && houseLouDongCount>0){ + regionHouseLDCount = regionHouseLDCount + houseLouDongCount + } + } + } + rows << [region, count, regionCount, regionHouseLDCount] + println region +" 完成" + } + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计") + } + + def sampleStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zHLevel1 = new AtomicLong() + def zHLevel2 = new AtomicLong() + def zHLevel3 = new AtomicLong() + def zHLevel4 = new AtomicLong() + def zHLevel5 = new AtomicLong() + + def fHLevel1 = new AtomicLong() + def fHLevel2 = new AtomicLong() + def fHLevel3 = new AtomicLong() + def fHLevel4 = new AtomicLong() + def fHLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zHLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fHLevel2.addAndGet(1) + }else if(results.size()<20){ + fHLevel3.addAndGet(1) + }else if(results.size()<50){ + fHLevel4.addAndGet(1) + }else if(results.size()>=50){ + fHLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zHLevel1.addAndGet(1) + + println d.city+"#"+d.region+"#"+d.name + }else{ + fCityDistTotal.addAndGet(1) + fHLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本") + } + + + def tradeStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zTLevel1 = new AtomicLong() + def zTLevel2 = new AtomicLong() + def zTLevel3 = new AtomicLong() + def zTLevel4 = new AtomicLong() + def zTLevel5 = new AtomicLong() + + def fTLevel1 = new AtomicLong() + def fTLevel2 = new AtomicLong() + def fTLevel3 = new AtomicLong() + def fTLevel4 = new AtomicLong() + def fTLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zTLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fTLevel2.addAndGet(1) + }else if(results.size()<20){ + fTLevel3.addAndGet(1) + }else if(results.size()<50){ + fTLevel4.addAndGet(1) + }else if(results.size()>=50){ + fTLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zTLevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fTLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交") + println "成交案例统计完成" + } + + def paimaiStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zPLevel1 = new AtomicLong() + def zPLevel2 = new AtomicLong() + def zPLevel3 = new AtomicLong() + def zPLevel4 = new AtomicLong() + def zPLevel5 = new AtomicLong() + + def fPLevel1 = new AtomicLong() + def fPLevel2 = new AtomicLong() + def fPLevel3 = new AtomicLong() + def fPLevel4 = new AtomicLong() + def fPLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zPLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fPLevel2.addAndGet(1) + }else if(results.size()<20){ + fPLevel3.addAndGet(1) + }else if(results.size()<50){ + fPLevel4.addAndGet(1) + }else if(results.size()>=50){ + fPLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zPLevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fPLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖") + println "拍卖案例统计完成" + } + def evalStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zELevel1 = new AtomicLong() + def zELevel2 = new AtomicLong() + def zELevel3 = new AtomicLong() + def zELevel4 = new AtomicLong() + def zELevel5 = new AtomicLong() + + def fELevel1 = new AtomicLong() + def fELevel2 = new AtomicLong() + def fELevel3 = new AtomicLong() + def fELevel4 = new AtomicLong() + def fELevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zELevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fELevel2.addAndGet(1) + }else if(results.size()<20){ + fELevel3.addAndGet(1) + }else if(results.size()<50){ + fELevel4.addAndGet(1) + }else if(results.size()>=50){ + fELevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zELevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fELevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估") + println "评估案例统计完成" + } + + def fileldStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + final CountDownLatch latch = new CountDownLatch(dists.size()) + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def alias_count = new AtomicLong() + def address_count = new AtomicLong() + def block_count = new AtomicLong() + def township_count = new AtomicLong() + def price_count = new AtomicLong() + def lng_count = new AtomicLong() + def finishBuildTime_count = new AtomicLong() + def developer_count = new AtomicLong() + def green_count = new AtomicLong() + def ratio_count = new AtomicLong() + def carsite_count = new AtomicLong() + def propertyManagement_count = new AtomicLong() + def managementFee_count = new AtomicLong() + def building_count = new AtomicLong() + def holed_count = new AtomicLong() + def car_holed_count = new AtomicLong() + def all_area_count = new AtomicLong() + def key_school_count = new AtomicLong() + def school_count = new AtomicLong() + def traffic_count = new AtomicLong() + def market_count = new AtomicLong() + def hospital_count = new AtomicLong() + def bank_count = new AtomicLong() + + def f_alias_count = new AtomicLong() + def f_address_count = new AtomicLong() + def f_block_count = new AtomicLong() + def f_township_count = new AtomicLong() + def f_price_count = new AtomicLong() + def f_lng_count = new AtomicLong() + def f_finishBuildTime_count = new AtomicLong() + def f_developer_count = new AtomicLong() + def f_green_count = new AtomicLong() + def f_ratio_count = new AtomicLong() + def f_carsite_count = new AtomicLong() + def f_propertyManagement_count = new AtomicLong() + def f_managementFee_count = new AtomicLong() + def f_building_count = new AtomicLong() + def f_holed_count = new AtomicLong() + def f_car_holed_count = new AtomicLong() + def f_all_area_count = new AtomicLong() + def f_key_school_count = new AtomicLong() + def f_school_count = new AtomicLong() + def f_traffic_count = new AtomicLong() + def f_market_count = new AtomicLong() + def f_hospital_count = new AtomicLong() + def f_bank_count = new AtomicLong() + + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + def index = new AtomicLong() + for(def i=0;i - if(!row['P'] && row['C']){ - def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null) + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null) if(jsonObject != null && jsonObject.precise!=-1) { - row['P'] = jsonObject.get("name") - row['Q'] = jsonObject.get("id") + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("id") + } + } + }) + } + + def matchMap(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + def propertyType = jsonObject.get("propertyType") + if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){ + row['Q'] = jsonObject.get("name") + row['R'] = new ObjectId().toString() + row['W'] = propertyType + + def arroundObj = FangjiaClientUtils.arroundPrice(city, address) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['F'] = priceObj.unitPrice + } + } + } } }) @@ -44,34 +93,51 @@ class PocTest { def addPrice(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['Q']){ - def dist = CommonDatautils.findDistrictById(row['Q']) + def commid = row['R'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid) if(dist){ def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) if(priceObj){ - row['E'] = priceObj.avg_price + row['F'] = priceObj.avg_price } } } }) } + def addArroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['F'] = priceObj.unitPrice + row['W'] = "周边均价" + } + } + }); + } + def standard(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['C']){ - def jsonObject = OpenApiClient.standard2Action(row['A'], row['C']) + def city = row['B'] + def address = row['D'] + if(address){ + println "[standard] address:"+address + def jsonObject = OpenApiClient.standard2Action(city, address) if(jsonObject){ println jsonObject - row['F'] = jsonObject.get("province") - row['G'] = jsonObject.get("city") - row['H'] = jsonObject.get("district") - row['I'] = jsonObject.get("township") - row['J'] = jsonObject.get("street") - row['K'] = jsonObject.get("streetNumber") - row['L'] = jsonObject.get("buildingNumber") - row['M'] = jsonObject.get("unitNumber") - row['N'] = jsonObject.get("floor") - row['O'] = jsonObject.get("roomNumber") + row['G'] = jsonObject.get("province") + row['H'] = jsonObject.get("city") + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("township") + row['K'] = jsonObject.get("street") + row['L'] = jsonObject.get("streetNumber") + row['M'] = jsonObject.get("buildingNumber") + row['N'] = jsonObject.get("unitNumber") + row['O'] = jsonObject.get("floor") + row['P'] = jsonObject.get("roomNumber") } } @@ -81,23 +147,27 @@ class PocTest { def addLoudongNum(path){ ExcelUtils.multiThreadingModify(path, { row -> - if (row['Q'] && row['L']) { - def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + def commid = row['R'] + def houseNum = row['M'] + def unitNum = row['N'] + println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum + if (commid && houseNum) { + def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum) if (jsonObject && jsonObject.result) { if(jsonObject.result.size()>=1){ def obj = jsonObject.result[0] - row['S'] = obj.id - row['R'] = obj.houseNumber + row['T'] = obj.id + row['S'] = obj.houseNumber } } - if(row['O']){ - def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if(row['P']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P']) if (jsonObject1 && jsonObject1.result) { - if(jsonObject.result.size()>=1){ - def obj = jsonObject.result[0] - row['U'] = obj.id - row['T'] = obj.roomNum + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + row['V'] = obj.id + row['U'] = obj.roomNum } } } @@ -111,12 +181,12 @@ class PocTest { rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"] for(def row : datas){ def data = [] - data << row['A'] data << row['B'] data << row['C'] data << row['D'] - if (row['Q'] && row['L']) { - def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + data << row['E'] + if (row['R'] && row['M']) { + def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N']) if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) { def obj = jsonObject.result[0] data << obj.houseNumber @@ -144,8 +214,8 @@ class PocTest { data << ""//景观 } - if(row['O']){ - def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if(row['P']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P']) if (jsonObject1 && jsonObject1.result) { if(jsonObject1.result.size()>=1){ def obj = jsonObject1.result[0] @@ -174,18 +244,20 @@ class PocTest { def rows = [] rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"] for(def d : datas){ - if(d['Q']){ - def dist = CommonDatautils.findDistrictById(d['Q']) + def commid = d['R'] + println "[addDetail] commid:"+commid + if(commid){ + def dist = CommonDatautils.findDistrictById(commid) if(dist){ def row = [] def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name) if(jsonObject) { - row << d['A'] row << d['B'] row << d['C'] + row << d['D'] row << jsonObject.get("name")//小区名称 - row << d['Q']//小区ID + row << commid//小区ID if(jsonObject.get("alias")){ row << jsonObject.get("alias").join(",").replace('"',"")//小区别名 }else{ @@ -207,7 +279,7 @@ class PocTest { row << jsonObject.get("buildingCount")//住宅幢数 row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数 row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比********* - row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积 + row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积 } @@ -271,28 +343,35 @@ class PocTest { rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"] def keys = [] for(def d : datas){ - if(d['Q']) { - def dist = CommonDatautils.findDistrictById(d['Q']) + def commid = d['R'] + println "[createPaimai] commid:"+commid + if(commid) { + def dist = CommonDatautils.findDistrictById(commid) if (dist) { def key = dist.city+"#"+dist.region+"#"+dist.name if(keys.contains(key)){ continue } keys << key - def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10) + def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100) if(jsonObject && jsonObject.result){ def results = jsonObject.result for(def r: results){ def row = [] - row << d['A'] row << d['B'] row << d['C'] + row << d['D'] row << r.matchName row << r.address row << r.houseNum row << r.roomNum row << r.area - row << r.startDate + if(r.startDate){ + row << r.startDate.substring(0, r.startDate.indexOf("T")) + }else{ + row << "" + } + row << r.count row << r.status row << r.tradeAvgPrice @@ -314,13 +393,15 @@ class PocTest { rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] def keys = [] for(def d : datas) { - if (d['Q']) { + def commid = d['R'] +// println "[createHouse] commid:"+commid + if (commid) { def key = d['Q'] if (keys.contains(key)) { continue } keys << key - def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20) + def jsonObject = OpenApiClient.houseList(commid,"20210501",20) if(jsonObject && jsonObject.result) { def results = jsonObject.result for (def r : results) { @@ -368,7 +449,7 @@ class PocTest { rows << row } - println d['Q'] +" "+results.size() + println d['R'] +" "+results.size() } } } @@ -381,13 +462,14 @@ class PocTest { rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] def keys = [] for(def d : datas) { - if (d['Q']) { - def key = d['Q'] - if (keys.contains(key)) { + def commid= d['R'] + println "[createEvaluation] commid:"+commid + if (commid) { + if (keys.contains(commid)) { continue } - keys << key - def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20) + keys << commid + def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20) if(jsonObject && jsonObject.result) { def results = jsonObject.result for (def r : results) { @@ -432,7 +514,7 @@ class PocTest { rows << row } - println d['Q'] +" "+results.size() + println d['R'] +" "+results.size() } } } @@ -446,13 +528,14 @@ class PocTest { rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] def keys = [] for(def d : datas) { - if (d['Q']) { - def key = d['Q'] - if (keys.contains(key)) { + def commid= d['R'] + println "[createDealPrice] commid:"+commid + if (commid) { + if (keys.contains(commid)) { continue } - keys << key - def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20) + keys << commid + def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20) if(jsonObject && jsonObject.result) { def results = jsonObject.result for (def r : results) { @@ -501,7 +584,7 @@ class PocTest { rows << row } - println d['Q'] +" "+results.size() + println d['R'] +" "+results.size() } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d6ab63366..26c6a3e15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -68,6 +68,10 @@ class CommonDatautils { public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl) + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl) + + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -704,7 +708,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","吴中路9弄小区") +// q.put("name","森望芳草绿岸") DBObject keys = new BasicDBObject(); if(fields){ @@ -827,6 +831,8 @@ class CommonDatautils { keys.put("out_pics", 1); keys.put("lat", 1); keys.put("lng", 1); + keys.put("lat2", 1); + keys.put("lng2", 1); keys.put("name_cat", 1); keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) @@ -1053,9 +1059,16 @@ class CommonDatautils { public static def queryDistrictById(def id){ + def dbo = null DBObject q = new BasicDBObject(); - q.put("_id", new ObjectId(id)); - def dbo = getDataStoreConnection().findOne(q); + try{ + q.put("_id", new ObjectId(id.trim().toString())); + dbo = getDataStoreConnection().findOne(q); + }catch(Exception e){ + println id + } + + return dbo; } @@ -3184,5 +3197,19 @@ class CommonDatautils { def avg_radios = (int)(radios.sum()/radios.size()) return avg_radios; } + + static def findLoudong(commid){ + def q = new BasicDBObject() + q.put("id_in_seaweed", commid) + def count = LOUDONG_LINE_COLL.count(q) + return count; + } + + static def findHouseLoudong(commid){ + def q = new BasicDBObject() + q.put("district_id", commid) + def count = HOUSE_LOUDONG_COLL.count(q) + return count; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy index 4628a207b..d5123668b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy @@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{ def run(){ def cityMap = CommonDatautils.getAllCityRegions() cityMap.each{city,regions-> - findPriceByCity(city, regions); + + findPriceByCity(city, regions); } } @@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{ q.put("region" , r.region) q.put("month", r.month) def mom = null - if(lastObj){ - mom = (r.price -lastObj.avg_price)/lastObj.avg_price - mom = DataHandleUtils.getDoubleRound(4, mom) + if(lastObj && lastObj.avg_price && r.price){ + mom = (r.price -lastObj.avg_price)/lastObj.avg_price + mom = DataHandleUtils.getDoubleRound(4, mom) } if(obj){ if(obj.avg_price != r.price){ @@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{ def findEsAvgPrice(city, region){ def detail = [:] - detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1]) + detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0]) detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]]) def must = [] @@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{ if(!priceValue){ priceValue = row['1-metric'] } - if(priceValue){ + if(priceValue && priceValue.value){ price = priceValue.value.toInteger() } result.price = price; def date = row.key_as_string.replace("-","").substring(0, 6) def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM") if(date!=curMonth){ - result.month = date.toInteger() - rets << result + if(date.toInteger()>200001 && result.price){ + result.month = date.toInteger() + rets << result + } + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy new file mode 100644 index 000000000..4601c3c60 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -0,0 +1,198 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeAppCrawl { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeAppCrawl(); + task.run(); +// task.run2() + } + + def run2(){ + def url = "https://m.anjuke.com/fz/community/" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def links = getRegion(url, driver) + for(def link : links){ + def doc = getDocument(url, driver) + def str = doc.toString().substring(doc.toString().indexOf("")) + if(str){ + str = str.substring(str.indexOf("} ("), str.indexOf("));")) + def arrs = str.split(",") + pritln arrs.size() + } + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + + def getRegion(url, driver){ + //def doc = getDocument(url, driver) + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") + header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") + header.put("referer","https://m.anjuke.com/fz/") + header.put("sec-fetch-dest","empty") + header.put("sec-fetch-mode","cors") + header.put("sec-fetch-site","same-origin") + header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1") + header.put("sw8-correlation","c291cmNl:YmVpZG91") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1") + + def doc = ABuYunUtil.doGet(url, null, header, "utf-8") + def links = [] + if(doc){ + def eles = JQueryUtils.select(".column2 a", doc) + for(def e : eles){ + def href = e.attr("href") + links << href + } + } + return links; + } + + + + + def run(){ + //"仓山":4071, + def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384] + map.each{region, area_id-> + for(def i=1;i<40;i++){ + def stop = false; + def all_stop = false + while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380 + //城口 + def flagMap = getInfo(stop, i, area_id) + stop = flagMap.stop + all_stop = flagMap.all_stop + } + if(all_stop){ + break; + } + } + } + } + + def getInfo(def stop, def i, def area_id){ + + def map = [:] + def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1" + + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") + header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") + header.put("referer","https://m.anjuke.com/fz/community/") + header.put("sec-fetch-dest","empty") + header.put("sec-fetch-mode","cors") + header.put("sec-fetch-site","same-origin") + header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1") + header.put("sw8-correlation","c291cmNl:YmVpZG91") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1") + + def ret = ABuYunUtil.doGet(url, null, null, "utf-8") + def all_stop = false; + if (ret) { + JSONObject jsonObj = JSONObject.fromObject(ret); + if(jsonObj && jsonObj.status=="ok" ){ + def data = jsonObj.data + if(data && data.total.toInteger()>0){ + def communities = data.communities + + println "【${i}】 "+communities.size() +" "+url + if(communities.size()>0){ + save("福州", communities) + Thread.sleep(10000) + stop = true; + }else{ + stop = true; + println url +" 返回0" + all_stop = true; + } + + }else{ + stop = false; + println url +" 无数据" + } + + + }else{ + stop = false; + Thread.sleep(5000) + } + } + + map.stop = stop + map.all_stop = all_stop; + return map; + } + + def save(city, def ret){ + def index = 0 + for(def o : ret){ + index++ + def obj = findIsExist(o.jumpAction) + if(!obj){ + def udate = new BasicDBObject() + o.each{k,v-> + udate.put(k,v) + } + udate.put("city", city) + udate.put("c_date", new Date()) + detail.save(udate) + println index+" [保存]"+o.jumpAction + }else{ + println index+" [已存在]"+o.jumpAction + } + + } + } + + def findIsExist(def url){ + def q = new BasicDBObject() + q.put("jumpAction",url) + return detail.findOne(q); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy index d4f3001b3..ab7aa4278 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy @@ -11,9 +11,15 @@ import org.openqa.selenium.By import org.openqa.selenium.WebDriver import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.remote.DesiredCapabilities +import org.openqa.selenium.remote.RemoteWebDriver class AnjukeDetailCrawl { + DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") + + DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url") + static main(args) { def task = new AnjukeDetailCrawl(); task.run(); @@ -21,9 +27,15 @@ class AnjukeDetailCrawl { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - def url = "https://chongqing.anjuke.com/community/?from=navigation" + def url = "https://chongqing.anjuke.com/" WebDriver driver = new ChromeDriver() +// DesiredCapabilities capabilities = DesiredCapabilities.chrome(); +// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL +// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities); + + driver.manage().deleteAllCookies() def regions = getRegions(driver, url) + println "区域链接: "+regions.size() getAllRegions(driver, regions) } @@ -31,44 +43,88 @@ class AnjukeDetailCrawl { def getAllRegions(driver, regions){ for(def url : regions){ - driver.get(url) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = getDocument(url, driver) def total = getTotal(doc) if(total) { if (total > 1250) { - + def links = getMoreLinks(url, driver) + println url +" "+links.size() + saveDetailLinks(links) +// parse(links,driver) } else { def links = getAllLinks(url, driver) - parse(links,driver) + println url +" "+links.size() + saveDetailLinks(links) +// parse(links,driver) } } } } + def saveDetailLinks(links){ + for(def url : links){ + def data = new BasicDBObject() + data.put("city","重庆") + data.put("url", url) + data.put("source", "安居客") + data.put("c_date", new Date()) + urlColl.save(data) + } + } + + def getMoreLinks(url, driver){ + def doc = Jsoup.parse(driver.getPageSource()); + def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc) + def links = [] + if(pricelinks){ + for(def link : pricelinks){ + if(link.text()!="不限"){ + def urlLink = link.attr("href") + getDocument(urlLink, driver) +// driver.get(urlLink) + links.addAll(getAllLinks(urlLink, driver)) + println urlLink +" "+links.size() + } + } + } + return links; + } + def getAllLinks(url, driver){ def stop = false def urls = [] - while(!stop) { - def doc = Jsoup.parse(driver.getPageSource()); + def doc = Jsoup.parse(driver.getPageSource()); + def total = getTotal(doc) + def page = getPage(total, 25) + def pages = [] + for(def i=1;i<=page;i++){ + pages << url.substring(0, url.size()-1)+"-p"+i+"/" + } + for(def pageUrl : pages){ + doc = getDocument(pageUrl, driver) def details = JQueryUtils.select(".list-cell a.li-row", doc) - for (def d : details) { - urls << d.attr("href") - } - def nextEle = JQueryUtils.select(".next", doc) - if (nextEle) { - driver.findElement(By.cssSelector(".next")).click() - }else{ - stop = true - } - if(urls.size()>20){ - stop = true + def link = d.attr("href") + if(!urls.contains(link)){ + urls << d.attr("href") + } } } return urls } + def getPage(def total, def pageSize){ + if(total && total.toInteger()>0){ + def page = total.toInteger()/pageSize; + if(page>(int)page){ + page = (int)page + 1 + return page + } + return (int)page; + } + return null + } def getTotal(doc){ def totalText = JQueryUtils.selectText(".total-info", doc) @@ -82,9 +138,19 @@ class AnjukeDetailCrawl { } def getRegions(driver, url){ +// def doc = getDocument(url, driver) driver.get(url) - Thread.sleep(3000) def doc = Jsoup.parse(driver.getPageSource()); + def els = JQueryUtils.select("a:contains(小区)", doc) + if(els){ + def href = els[0].attr("href") + doc = Jsoup.parse(driver.getPageSource()); + // driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click() +// driver.findElement(By.linkText("小区")).click() + doc = getDocument(href, driver) + } + + // def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) // if(loginIfm){ // def href = loginIfm[0].attr("src") @@ -107,23 +173,35 @@ class AnjukeDetailCrawl { def regionEles = JQueryUtils.select(".region-parents li a", doc) def urls = [] for(def e : regionEles){ - urls << e.attr("href") + if(e.text()!="不限"){ + urls << e.attr("href") + } } return urls; } + def getNontCheckPage(url, driver, doc){ + def stop = false; + while(!stop){ + if(doc){ + + } + } + return doc; + } + def parse(def links, driver){ def rets = [] for(def link : links){ def district = parsing(link, driver) if(district){ - rets << district + // rets << district - if (rets.size() % 100 == 0) { - def add= saveData(rets) - println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - rets = [] - } + // if (rets.size() % 100 == 0) { + saveObj(district) + println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + // rets = [] + // } } } if (rets.size()> 0) { @@ -136,33 +214,61 @@ class AnjukeDetailCrawl { def saveData(def results){ def add = 0 for(def ret : results){ - def q = new BasicDBObject() - q.put("url", ret.url) - def dbo = detailColl.findOne(q); - - DBObject obj = new BasicDBObject(); - ret.each{k,v-> - obj.put(k,v) - } - - if(dbo){ - obj.m_date = new Date() - detailColl.update(q, new BasicDBObject('$set', obj)) - }else{ - add++ - obj.c_date = new Date() - obj.m_date = new Date() - try{ - detailColl.save(obj) - }catch(Exception e){ - println obj.url - } - } + saveObj(ret) } - return add } + def saveObj(def ret){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = detailColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + detailColl.update(q, new BasicDBObject('$set', obj)) + }else{ + + obj.c_date = new Date() + obj.m_date = new Date() + try{ + detailColl.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + def parsing(url, driver){ @@ -191,10 +297,9 @@ class AnjukeDetailCrawl { def infoDiv = JQueryUtils.select(".info-list", doc) district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) - district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) - district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) - district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) - + district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text()) + district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) + district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() if (houseCountStr) { houseCountStr = houseCountStr.replace("户", "") @@ -202,9 +307,13 @@ class AnjukeDetailCrawl { district.house_count = houseCountStr.toInteger() } } - - district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) + district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) + district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text()) + + district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) + district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text()) + def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) if(carSiteStr){ if(carSiteStr.contains("(")){ @@ -214,16 +323,19 @@ class AnjukeDetailCrawl { district.car_site = carSiteStr } } - district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) - district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) - district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) - district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) - district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text() + district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) + + district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text()) + + district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) + district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) district.url = url district.c_date = new Date() district.m_date = new Date() + district.crawl_type = "chrome" district.root = "anjuke.com" - stop = true; }else{ Thread.sleep(3000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy new file mode 100644 index 000000000..f990711be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class AnjukeDistrictMatch { + + static main(args) { + def task = new AnjukeDistrictMatch(); + task.run(); +// task.run1() + } + + def run1(){ + def regionPointsMap = findRegionRange("重庆") + def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + } + def run(){ + + + + def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx" + ExcelUtils.multiThreadingModify(filePath,{row-> + if(row['A'] && !row['J']){ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def address = row['E'] +// def flag = "别名匹配" + + +// def dist = matchDist(city, null, name) +// if(!row['J'] && name.contains("(")){ +// name = name.replace("(${region})","").replace("(","").replace(")","") +// flag = "去括号匹配" +// dist = matchDist(city, region, name) +// } +// if(!dist){ +// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") +// flag = "映射别名匹配" +// } +// if(!dist) { +// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") +// if(rets && rets.size()==1){ +// dist = rets[0] +// } +// } + +// +// if(!dist){ +// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district") +// flag = "地址匹配" +// if(dists.size()>1){ +// println address +" 地址相同:"+dists.size() +// }else{ +// if(dists.size()==1){ +// dist = dists[0] +// } +// } +// } + +// if(dist){ +// row['J'] = flag +// row['K'] = dist._id.toString() +// row['L'] = dist.region +// row['M'] = dist.name +// row['N'] = dist.address +// } + + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) + if(jsonObject != null && jsonObject.precise!=-1) { +// row['J'] = "匹配接口" +// row['K'] = jsonObject.get("id") +// row['L'] = jsonObject.get("district") +// row['M'] = jsonObject.get("name") +// row['N'] = jsonObject.get("address") + row['M'] = jsonObject.get("name") + } + } + }) + } + + def matchDist(city, region, name){ + def dist = CommonDatautils.findDistrict(city, region, name, "district") + return dist; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy new file mode 100644 index 000000000..1954d135c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy @@ -0,0 +1,628 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +class AnjukeToDB { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeToDB(); +// task.run(); +// task.addNoMatch() + task.updateEstateType() +// task.gaodeRangeData() + } + + def updateEstateType(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1) + for(def row : datas){ + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj) { + def type = getEstatType(obj.extend.type) + def commId = row['L'] + if(!commId){ + commId = row['G'] + } + if(type){ + if(type=="别墅"){ + def udata = new BasicDBObject() + udata.put("estate_type2", type) + udata.put("m_date", new Date()) + println commId + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata)) + } + }else{ +// def udata = new BasicDBObject() +// udata.put("status", 1) +// udata.put("name_cat",11) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata)) + + } + } + } + } + + def gaodeRangeData(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1) + def index = 0 + for(def d : datas){ + index++ + println index + def personName = d['AB'] + if(personName){ + def commid = d['J'] + if(commid){ + def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district") + if(obj){ + updateDist(d, obj._id.toString()) + } + }else{ + saveGaode(d) + } + }else{ + def commid = d['J'] + updateDist(d, commid) + } + + } + } + + def saveGaode(d){ + def oldName = d['E'] + def distName = nameFileter(d['C'], oldName) + def udata = new BasicDBObject() + def alias = [] + + def aliasStr = d['F'] + if(aliasStr){ + def arrs = aliasStr.split("/") + for(def s : arrs){ + alias << s + } + } + if(alias.size()>0){ + udata.put("alias", alias) + } + + def address = AddressUtil.formatAddress("重庆", d['G']) + if(address){ + udata.put("address", address) + } + if(d['N']){ + udata.put("virescence_percent", d['N']) + } + + if(d['O']){ + udata.put("estate_charge", d['O']) + } + if(d['Q']){ + udata.put("cubage_percent", d['Q']) + } + if(d['R']){ + udata.put("developer", d['R']) + } + + if( d['S']){ + udata.put("build_area", d['S']) + } + + if(d['T']){ + def t = d['T'].substring(0, 3) + udata.put("build_date", t) + } + + if(d['V'] && d['V'].isNumber()){ + udata.put("car_site_count", d['V'].toInteger()) + } + + if(d['Y']){ + def build_type = getBuildType2(d['Y']) + if(build_type){ + udata.put("build_type", build_type) + } + } + + if(d['Z']){ + udata.put("property_life", d['Z']) + } + + def lat = null + def lng = null + if(d['H']){ + lat = d['H'] + lng = d['I'] + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + Long dlat = lat.toDouble() * 10000000000L + Long dlng = lng.toDouble() * 10000000000L + udata.put("lat", dlat) + udata.put("lng", dlng) + } + def type = "普通住宅" + if(distName.contains("别墅")) { + type = "别墅" + }else if(distName.contains("商住")){ + type = "商住" + } + + if(udata){ + udata.put("name_cat",0) + udata.put("status",0) + udata.put("cat","district") + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("city", "重庆") + udata.put("region",d['C']) + udata.put("name", distName) + udata.put("estate_type2",type) + + def markMap = [:] + markMap.put("data_source", "机器入库") + udata.put("mark_map",markMap) + DbUtils.seaweed.save(udata); + + def stop = false + while (!stop){ + if(udata._id.toString()){ + addPrice(udata) + stop = true; + } + } + } + } + + def updateDist(d, commid){ + + if(commid){ + def dist = CommonDatautils.queryDistrictById(commid) + if(dist){ + def oldName = d['E'] + def distName = nameFileter(d['C'], oldName) + def udata = new BasicDBObject() + if(!distName.contains("号")){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(!alias.contains(distName) && dist.name!=distName){ + alias << distName + } + def aliasStr = d['F'] + if(aliasStr){ + def arrs = aliasStr.split("/") + for(def s : arrs){ + alias << s + } + } + if(alias.size()>0 && alias!=dist.alias){ + udata.put("alias", alias) + } + } + + if(dist.address && !dist.address.contains("号")){ + def address = AddressUtil.formatAddress("重庆", d['G']) + if(address){ + udata.put("address", address) + } + } + + if(!dist.virescence_percent && d['N']){ + udata.put("virescence_percent", d['N']) + } + + if(!dist.estate_charge && d['O']){ + udata.put("estate_charge", d['O']) + } + if(!dist.cubage_percent && d['Q']){ + udata.put("cubage_percent", d['Q']) + } + if(!dist.developer && d['R']){ + udata.put("developer", d['R']) + } + + if(!dist.build_area && d['S']){ + udata.put("build_area", d['S']) + } + + if(!dist.build_date && d['T']){ + if(d['T'].size()>=4){ + def t = d['T'].substring(0, 4) + udata.put("build_date", t) + } + + } + + if(!dist.car_site_count && d['V'] && d['V'].isNumber()){ + udata.put("car_site_count", d['V'].toInteger()) + } + + if(!dist.build_type && d['Y']){ + def build_type = getBuildType2(d['Y']) + if(build_type){ + udata.put("build_type", build_type) + } + } + + if(!dist.property_life && d['Z']){ + udata.put("property_life", d['Z']) + } + + def lat = null + def lng = null + if(d['H']){ + lat = d['H'] + lng = d['I'] + + def slat = dist.lat/10000000000.0 + def slng = dist.lng/10000000000.0 + + def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat); + if(distance>1000){ + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + Long dlat = lat.toDouble() * 10000000000L + Long dlng = lng.toDouble() * 10000000000L + udata.put("lat", dlat) + udata.put("lng", dlng) + } + + } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改] ${dist.name}"+ udata + } + } + } + } + + def addNoMatch(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1) + for(def d : datas){ + add(d) + } + } + def run(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1) + for(def data : datas){ + if(data['K']=="芝芝"){ + def flag = data['M'] + if(flag && flag=="新增"){ + add(data) + }else{ + update(data) + } + } + } + } + + + def add(def row){ + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj){ + def type = getEstatType(obj.extend.type) + if(type){ + def city = obj.base.cityName + def region = obj.base.areaName + def lat2 = null + def lng2 = null + def lat = null + def lng = null + if(obj.base.lat){ + lat2 = obj.base.lat + lng2 = obj.base.lng + + JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu") + if(convertObj.getInt("status") == 1) { + String lo = convertObj.get("locations") + lng = Double.parseDouble(lo.split(",")[0]) + lat = Double.parseDouble(lo.split(",")[1]) + } + } + + def distName = nameFileter(obj.base.areaName, obj.base.name) + + def buildTypeStr = getBuildType(obj.base.buildTypeStr) + + def address = AddressUtil.formatAddress(obj.base.city, obj.base.address) + + def dist = CommonDatautils.findDistrict(city, region, distName, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",region) + data.put("name", distName) + data.put("estate_type2",type) + if(address){ + data.put("address", obj.base.address) + } + + if(lat){ + Long alat = lat*10000000000L + Long alng = lng*10000000000L + data.put("lat", alat) + data.put("lng", alng) + Long alat2 = lat2.toDouble()*10000000000L + Long alng2 = lng2.toDouble()*10000000000L + data.put("lat2",alat2) + data.put("lng2",alng2) + } + data.put("finish_building_date",obj.extend.completionTime) + if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){ + data.put("household_count",obj.extend.totalHouseHoldNum.toInteger()) + } + data.put("build_area", obj.extend.totalArea) + + data.put("virescence_percent", obj.extend.landscapingRatio) + data.put("build_type", buildTypeStr) + data.put("business_quarter",obj.base.tradingAreaName) + + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + DbUtils.seaweed.save(data); + + def stop = false + while (!stop){ + if(data._id.toString()){ + addPrice(data) + stop = true; + } + } + + } + }else{ + println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type + } + }else{ + println anjukeId+" 没找到" + } + + } + def update(def row){ + def commId = row['L'] + if(!commId){ + commId = row['G'] + } + + def dist = CommonDatautils.queryDistrictById(commId) + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj && dist){ + def udata = new BasicDBObject() + + def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address) + if(address){ + if(!dist.address){ + if(obj.base.address){ + udata.put("address",obj.base.address) + } + }else{ + if(dist.address.contains("号")){ + if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){ + def addr_alias = dist.addr_alias + if(!addr_alias){ + addr_alias = [] + } + if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){ + addr_alias << obj.base.address + } + if(addr_alias!=dist.addr_alias && addr_alias.size()>0){ + udata.put("addr_alias", addr_alias) + } + } + }else{ + udata.put("address",obj.base.address) + } + } + } + + def distName = nameFileter(obj.base.areaName, obj.base.name) + if(!distName.contains("号")){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(alias.contains(distName)){ + alias << distName + } + if(alias.size()>0 && alias!=dist.alias){ + udata.put("alias", alias) + } + } + + if(!dist.build_type){ + def buildTypeStr = getBuildType(obj.base.buildTypeStr) + if(buildTypeStr){ + udata.put("build_type",buildTypeStr) + } + + } + if(!dist.household_count){ + if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){ + udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger()) + } + } + if(!dist.finish_building_date){ + if(obj.extend.completionTime){ + udata.put("finish_building_date",obj.extend.completionTime) + } + } + + if(!dist.virescence_percent){ + if(obj.extend.landscapingRatio){ + udata.put("virescence_percent", obj.extend.landscapingRatio) + } + } + if(!dist.business_quarter){ + if(obj.base.tradingAreaName){ + udata.put("business_quarter",obj.base.tradingAreaName) + } + } + def lat2 = null + def lng2 = null + if(dist.lat2){ + lat2 = dist.lat2/10000000000.0 + lng2 = dist.lng2/10000000000.0 + } + if(dist.lat && !lat2){ + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3) + lng2 = baidu.x + lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + } + + if(obj.base.lat) { + def alat2 = obj.base.lat.toDouble() + def alng2 = obj.base.lng.toDouble() + def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2); + if(distance>1000){ + println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................" + + def lng = null + def lat = null + JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu") + if(convertObj.getInt("status") == 1) { + String lo = convertObj.get("locations") + lng = Double.parseDouble(lo.split(",")[0]) + lat = Double.parseDouble(lo.split(",")[1]) + } + + Long blat2 = alat2 *10000000000L + Long blng2 = alng2 *10000000000L + Long gLat2 = lat *10000000000L + Long gLng2 = lng *10000000000L + + udata.put("lat2", blat2) + udata.put("lng2", blng2) + udata.put("lat", gLat2) + udata.put("lng", gLng2) + } + } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改] ${dist.name}"+ udata + } + + } + } + + def findAnjukeDistrictDetail(anjukeId){ + def q = new BasicDBObject() + q.put("_id",new ObjectId(anjukeId)) + return detail.findOne(q) + } + + def nameFileter(def region, def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + name = name.replace("(${region})","").replace("(","").replace(")","") + return name; + } + + def getEstatType(type){ + if(type){ + if(type=="公寓住宅"){ + type = "普通住宅" + }else if(type.contains("商住")){ + type = "商住" + }else if(type.contains("别墅")){ + type = "别墅" + }else{ + type = null + } + } + return type + } + + def getBuildType(def buildType){ + def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"] + def list = [] + if(buildType){ + def arrs = buildType.split("\\|") + for(def s : arrs){ + if(types.contains(s)){ + list << s + } + } + if(list){ + return list.join(",") + } + } + return null + } + + def getBuildType2(def buildType){ + def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"] + def list = [] + if(buildType){ + def arrs = buildType.split(",| ") + for(def s : arrs){ + if(types.contains(s)){ + list << s + } + } + if(list){ + return list.join(",") + } + } + return null + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index a6b739898..a97cb7669 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -358,13 +358,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("重庆")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap //cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy index 3ea9f1d4a..e11e01e8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -29,7 +29,7 @@ class JdPaimaiCrawl { task.run(); } def run(){ - def existsIds = findAllIds() + def existsIds = []//findAllIds() println "已存在:"+existsIds.size() System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); @@ -43,78 +43,80 @@ class JdPaimaiCrawl { def getList(driver){ def list = [] - for(def i=1;i<=32;i++){ - if(i==8){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" - }else if(i==5){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" - }else if(i==17){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" - }else if(i==19){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" - }else{ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" - } - } + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268" +// for(def i=1;i<=32;i++){ +// if(i==8){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" +// }else if(i==5){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" +// }else if(i==17){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" +// }else if(i==19){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" +// }else{ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" +// } + +// } def links = [] for(def url : list){ driver.get(url) @@ -430,10 +432,10 @@ class JdPaimaiCrawl { def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc) if(s1) { - def start = s1.indexOf("法院将于") + def start = s1.indexOf("将于") def end = s1.indexOf("止") if(start>-1 && end>-1 && end>start){ - s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止")) } def dateRangs = DataUtil.getDateRange(s1) if(dateRangs && dateRangs.size()==2) { @@ -476,6 +478,11 @@ class JdPaimaiCrawl { } } + + if(!result.startDate){ + def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","") + result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日") + } if(result.finishDate){ result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6f99480ab..cae68c08d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,10 +34,30 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); - task.run() +// task.run() // task.run1() - task.addDealRegion(); - task.standardRegion() +// task.addDealRegion(); +// task.standardRegion() + task.updadteDealPriceField() + } + + def updadteDealPriceField(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def city : citys){ + index ++ + def q = new BasicDBObject() + q.put("city", city) + q.put("hall", "0") + def count = DEAL_PRICE_COLL.count(q) + println "[${index}] "+city +" "+count + if(count && count>0){ + def udata = new BasicDBObject() + udata.put("hall", 0) + DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true); + println "[修改]"+city +" "+ count + } + } } def run1(){ @@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["长春","吉林"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index c7451c5af..6a50bd201 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -30,8 +30,8 @@ class PaimaiTest { def task = new PaimaiTest(); // task.run(); // task.stat() -// task.updatePaimai() - task.updateFieldPaimai() + task.updatePaimai() +// task.updateFieldPaimai() } def updateFieldPaimai(){ @@ -42,8 +42,8 @@ class PaimaiTest { for(def obj : dbcursor) { index++ println index - if(obj.depositPrice){ - obj.put("depositPrice", obj.depositPrice.toInteger()) + if(obj.address && obj.address.contains("可贷款")){ + obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款"))) paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) } } @@ -51,11 +51,22 @@ class PaimaiTest { def updatePaimai(){ def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) def index =0 for(def d : dbcursor){ index++ println index + if(!d.url.contains("jd.com")){ + continue + } + + if(!d.proince.contains("重庆")){ + continue + } + + println d.url + def query = new BasicDBObject() query.put("url", d.url) def dbo = paimai43_COLL.findOne(query); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index 9fbafb2ae..2a231a42c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -42,8 +42,41 @@ class CqfdcMatch { // task.export2() // task.matchLoudong() - task.addLoudong() +// task.addLoudong() // task.addFdcLoudong() + task.updateHouseLoudong() + } + + def updateHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + + def keys = new BasicDBObject() + keys.put("room_num", 1) + keys.put("house_floor", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def index = 0 + for(def d : dbcursor){ + index++ + def room_num = d.room_num + def house_floor = d.house_floor + try{ + if(room_num && house_floor && room_num.isNumber()){ + if(house_floor.toInteger()>0){ + if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){ + def newValue = house_floor +"-"+room_num + println "[${index}] "+room_num +" "+house_floor+" "+newValue + def udata = new BasicDBObject() + udata.put("room_num",newValue) + HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false); + } + } + } + }catch(Exception e){ + println d._id.toString() + } + + } } def export(){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 97af5b6dc..d7222f3d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "58同城"// + def source = "贝壳找房"// def citys = findCrawlCity(source, 202202); for(def city : citys){ - def count1 = findCount(source, city, 202202); - def count2 = findCount(source, city, 202203); + def count1 = findCount(source, city, 202203); + def count2 = findCount(source, city, 202204); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index c676385ed..4784e512e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); -// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] + def citys = ["上海","北京","广州","天津","武汉","南京","青岛","合肥","西安","沈阳","宁波","佛山","潍坊","济南","大连","兰州","石家庄","长春","温州","乌鲁木齐","太原","惠州","嘉兴","南通","南昌","烟台","临沂","呼和浩特","厦门","海口","常州","中山","绍兴","淄博","洛阳","珠海","金华","威海","德阳","唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","衡阳","滨州","张家口","赣州","商丘","安庆","湖州","枣庄","聊城","阜阳","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳","蚌埠","九江","临汾","大同","六安","淮南","宿州","三亚","玉溪","六盘水","揭阳","汕尾","恩施","盘锦","伊犁","玉林","随州","抚州","景德镇","四平","丽水","安顺","毕节","天水","阜新","通化","攀枝花","吕梁","阿克苏","新余","松原","楚雄","鹤壁","忻州","本溪","拉萨","三门峡","黔东南","兴安盟","雅安","仙桃","乌海","贵港","安康","白城","平凉","庆阳","延安","潮州","白银","黔西南","丽江","吴忠","防城港","喀什","哈尔滨","钦州","鸡西","石嘴山","哈密","萍乡","鹰潭","武威","保山","铜仁","普洱","酒泉","宜昌","河池","朔州","百色","鹤岗","张家界","湘西","张掖","定西","昭通","济源","中卫","黑河","克拉玛依","铜川","辽源","云浮","博尔塔拉","固原","白山","广元","贺州","潜江","天门","伊春","石河子","临沧","榆林","临夏","来宾","成都","双鸭山","梧州","嘉峪关","海东","昌吉"] + // for(def city : citys){ // def str = cityList.get(city) // for(def i=0;i<1;i++){ @@ -209,7 +210,7 @@ class Esf58Crawl extends BaseBatch{ cityList.each{city, str-> for(def i=0;i<1;i++){ -// if(citys.contains(city)){ + if(citys.contains(city)){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); if(districts){ def arrs = str.split("\\|") @@ -219,7 +220,7 @@ class Esf58Crawl extends BaseBatch{ addCrawlKeys(city); addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); } -// } + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index fec88b8c5..b620f3098 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"] + def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -26,7 +26,7 @@ class SynchDistrictPrice { def run(){ - def citys = CommonUtils.findAllCity() + def citys = ["重庆"]//CommonUtils.findAllCity() sourceMap.each { k, v -> for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 29c5f5f4b..1a45be219 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -236,10 +236,10 @@ public class OpenApiClient { params.put("houseNumber", houseNum); params.put("unitNumber", unitNum); params.put("token", TOKEN); - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8") println("${id} ${houseNum} 程序调用异常") Thread.sleep(6000) } @@ -266,10 +266,10 @@ public class OpenApiClient { params.put("unitNumber", unitNum); params.put("roomNumber", roomNum); params.put("token", TOKEN); - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") println("${id} ${houseNum} 程序调用异常") Thread.sleep(6000) } @@ -687,7 +687,7 @@ public class OpenApiClient { params.put("region", region) params.put("name", name) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result @@ -697,13 +697,14 @@ public class OpenApiClient { return null } - static JSONObject searchPaimais(String city, String region, String name, limit) { + static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) { Map params = new HashMap(); params.put("token", TOKEN); params.put("city", city) params.put("district", region) params.put("name", name) - params.put("limit", "10") + params.put("startDate", startDate) + params.put("limit", "100") try { String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8") JSONObject result = JSONObject.fromObject(content); @@ -722,7 +723,7 @@ public class OpenApiClient { params.put("startDate", startDate) params.put("limit", limit.toString()) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { @@ -738,7 +739,7 @@ public class OpenApiClient { params.put("startTime", startDate) params.put("pageSize", limit.toString()) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { @@ -755,7 +756,7 @@ public class OpenApiClient { params.put("startDate", startDate) params.put("limit", limit.toString()) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { From 04e57b12f3c549b2e47c3f422af94f963e6d2246 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 22 May 2022 14:18:17 +0800 Subject: [PATCH 04/12] =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 4978bd962..8d7bf9732 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) From 195916001234f20a5f07f887446b363e49f781fd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 22 May 2022 15:54:50 +0800 Subject: [PATCH 05/12] update --- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- .../script/ccbevaluation/TradeStatLog.groovy | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index e262bd7c5..d1cc4209b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202201 - def endDate = 202203 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3ed13ebfc..3b3e6a7c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-01-01" - def date = "2022-02-01" - def endDate = "2022-03-01" - def newDate = "2022-04-01" + def startDate = "2022-02-01" + def date = "2022-03-01" + def endDate = "2022-04-01" + def newDate = "2022-05-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据") println "表格生成完成" } From 9312b352b0abc51323b6753e77a30ea99a7bd6e4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 22 May 2022 18:49:07 +0800 Subject: [PATCH 06/12] =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 39 +++++++++++++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 060914122..af8723b42 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -38,8 +38,8 @@ class ExportOracleData { exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-03-19") - exportBigdataDeal("2022-03-01") + exportBigdataForsale("2022-04-19") + exportBigdataDeal("2022-04-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 26c6a3e15..af552d3d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -590,8 +590,8 @@ class CommonDatautils { } - public static def findDistrict(String city, String region, String name, String cat){ - + public static def findDistrictCount(String city, String region, String name, String cat){ + DBObject q = new BasicDBObject(); q.put("city", city); if(region){ @@ -602,7 +602,24 @@ class CommonDatautils { q.put("cat", cat) q.put("status", 0) - + def count = getDataStoreConnection().count(q); + return count; + } + + + public static def findDistrict(String city, String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + if(region){ + q.put("region", region); + } + + q.put("name", name); + q.put("cat", cat) + q.put("status", 0) + + DSCursor cursor = getDataStoreConnection().find(q); if(cursor){ for(def obj : cursor){ @@ -613,6 +630,7 @@ class CommonDatautils { } + public static def findDistrictList(String city, String region, String name, String cat){ DBObject q = new BasicDBObject(); @@ -837,12 +855,18 @@ class CommonDatautils { keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) keys.put("sid", 1) - DSCursor cursor = getDataStoreConnection().find(q, keys); - if(cursor){ - for(def obj : cursor){ - return obj; + try{ + DSCursor cursor = getDataStoreConnection().find(q, keys); + if(cursor){ + for(def obj : cursor){ + return obj; + } } + }catch(Exception e){ + println "error_id:"+id + e.printStackTrace() } + return null; } @@ -2398,6 +2422,7 @@ class CommonDatautils { q.put("block", block); } q.put("cat", cat); + q.put("status",0) def cur = getDataStoreConnection().find(q); return cur.count(); } From 13a146e4c6962a6402fbe98db81fe4e46a0b68a2 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 10:30:42 +0800 Subject: [PATCH 07/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=80=E6=88=BF?= =?UTF-8?q?=E4=B8=80=E4=BB=B7=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 96a414024..eec63ef82 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -178,6 +178,7 @@ public class PropertyController { private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; private CommunityPoiServiceImpl communityPoiService; + private EvaluateFactorService evaluateFactorService; private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; @@ -1170,6 +1171,22 @@ public class PropertyController { mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); return mav; } + // 调整面积不同面积价格 + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + evaluation.setAvgPrice(avgPrice); + evaluation.setTotalPrice((int) (avgPrice * evaluation.getArea())); + } Map districtPriceMap = new LinkedHashMap(); districtPriceMap.put("city", district.getCity()); districtPriceMap.put("district", district.getRegion()); @@ -5631,4 +5648,8 @@ public class PropertyController { public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) { this.communityPoiService = communityPoiService; } + + public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) { + this.evaluateFactorService = evaluateFactorService; + } } From 70338124281a42f14e4502e85de8b48351bc912a Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 13:20:19 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=80=E6=88=BF?= =?UTF-8?q?=E4=B8=80=E4=BB=B7=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 115 ++++++++++++------ 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index eec63ef82..69c1eb263 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -397,7 +397,7 @@ public class PropertyController { /** * 利用百度API把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -419,7 +419,7 @@ public class PropertyController { /** * 利用算法模板把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -611,7 +611,7 @@ public class PropertyController { /** * 判断小区是否有经纬度 - * + * * @param district * @return */ @@ -622,7 +622,7 @@ public class PropertyController { /** * 调用匹配接口匹配小区 - * + * * @param city * @param address * @param region @@ -1000,6 +1000,32 @@ public class PropertyController { } } + // 调整面积不同面积价格 + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + + } + if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0 && (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) { Map districtPriceMap = new HashMap(); @@ -1172,20 +1198,29 @@ public class PropertyController { return mav; } // 调整面积不同面积价格 - DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); - int orgAvgPrice = 0; - if (districtStatBo != null) { - if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { - orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); - } else if (districtStatBo.getDistrictStat() != null) { - orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } } - } - if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { - int factorPrice = evaluation.getAvgPrice(); - int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); - evaluation.setAvgPrice(avgPrice); - evaluation.setTotalPrice((int) (avgPrice * evaluation.getArea())); + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + } Map districtPriceMap = new LinkedHashMap(); districtPriceMap.put("city", district.getCity()); @@ -1512,7 +1547,7 @@ public class PropertyController { return mav; } } - + /** * 四川锦程 定制接口 多建筑年代 * @param model @@ -1624,7 +1659,7 @@ public class PropertyController { districtPriceMap.put("buildYear", matcher3.group()); } } - + // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -2137,7 +2172,7 @@ public class PropertyController { /** * 小区租金查询接口 - * + * * @param model * @return */ @@ -2616,7 +2651,7 @@ public class PropertyController { /** * 批量获取小区的均价,均价走势数据 - * + * * @param request * @return */ @@ -2758,7 +2793,7 @@ public class PropertyController { /** * 批量获取小区的配套数据,最多12个小区 - * + * * @param request * @return */ @@ -2903,7 +2938,7 @@ public class PropertyController { /** * 获取JSON参数中的num参数值 - * + * * @param jo * @return */ @@ -2928,7 +2963,7 @@ public class PropertyController { /** * 获取JSON参数中的radius值 - * + * * @param jo * @return */ @@ -3736,7 +3771,7 @@ public class PropertyController { /** * 解析出关键字中的小区名 - * + * * @param keyword * @return */ @@ -3853,7 +3888,7 @@ public class PropertyController { /** * 小区统计分析接口 - * + * * @param model * @return */ @@ -3938,7 +3973,7 @@ public class PropertyController { /** * 小区交通接口 - * + * * @param model * @return */ @@ -3996,7 +4031,7 @@ public class PropertyController { /** * 小区周边接口 - * + * * @param model * @return */ @@ -4055,7 +4090,7 @@ public class PropertyController { /** * 相同小区相似户型房源挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4095,7 +4130,7 @@ public class PropertyController { /** * 周边小区相似户型挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4146,7 +4181,7 @@ public class PropertyController { /** * 小区周边一公里均价 - * + * * @author wenyan * @param model * @return @@ -4169,7 +4204,7 @@ public class PropertyController { /** * 相同小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4212,7 +4247,7 @@ public class PropertyController { /** * 周边小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4275,7 +4310,7 @@ public class PropertyController { /** * 室号查询接口 - * + * * @param model * @return */ @@ -4311,7 +4346,7 @@ public class PropertyController { /** * 楼栋楼层信息查询接口 - * + * * @param model * @return */ @@ -4336,7 +4371,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4381,7 +4416,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4834,7 +4869,7 @@ public class PropertyController { /** * 房产交易记录查询 - * + * * @param model * @return */ @@ -4894,7 +4929,7 @@ public class PropertyController { /** * 房产拍卖记录 - * + * * @param model * @return */ @@ -4955,7 +4990,7 @@ public class PropertyController { /** * 房产近期评估次数 - * + * * @param model * @return */ From 1648838af9abb97fb78a8402e7f8fd202e895c16 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 13:30:18 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BD=AE=E4=BF=A1?= =?UTF-8?q?=E5=BA=A6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mongo/open/dao/CommunityBasePriceDao.java | 9 +++ .../dao/impl/CommunityBasePriceDaoImpl.java | 10 +++ .../mongo/open/model/CommunityBasePrice.java | 74 +++++++++++++++++++ .../open/controllers/PropertyController.java | 21 ++++-- .../open/CommunityBasePriceService.java | 9 +++ .../impl/CommunityBasePriceServiceImpl.java | 22 ++++++ 6 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java new file mode 100644 index 000000000..8275f68f8 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java new file mode 100644 index 000000000..c89477b09 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public class CommunityBasePriceDaoImpl extends MongoDaoImpl implements CommunityBasePriceDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java new file mode 100644 index 000000000..9321bd542 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java @@ -0,0 +1,74 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +/** + * 小区建行置信度信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "business.community_base_price_gj") +public class CommunityBasePrice implements MongoEntity { + + private static final long serialVersionUID = -8010010773207622878L; + + private String id; + private String city; + private String region; + private String name; + private Double credit; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "credit") + public Double getCredit() { + return credit; + } + + public void setCredit(Double credit) { + this.credit = credit; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 69c1eb263..0a7df7866 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -31,6 +31,7 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; import com.fangjia.dal.mongo.model.*; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; import com.fangjia.dal.mongo.open.model.CommunityPoi; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; @@ -177,7 +178,8 @@ public class PropertyController { private SeawaterDao seawaterDao; private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; - private CommunityPoiServiceImpl communityPoiService; + private CommunityPoiService communityPoiService; + private CommunityBasePriceService communityBasePriceService; private EvaluateFactorService evaluateFactorService; private FileDao fileDao = FileDao.getInstance(); @@ -1235,6 +1237,13 @@ public class PropertyController { // } districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); + // 新增置信度 + CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId()); + if (communityBasePrice != null) { + districtPriceMap.put("credit", communityBasePrice.getCredit()); + } else { + districtPriceMap.put("credit", evaluation.getCredit()); + } List names = new ArrayList(); names.add(district.getName()); if(district.getAlias() != null){ @@ -5676,15 +5685,15 @@ public class PropertyController { this.rentDealPriceService = rentDealPriceService; } - public CommunityPoiServiceImpl getCommunityPoiService() { - return communityPoiService; - } - - public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) { + public void setCommunityPoiService(CommunityPoiService communityPoiService) { this.communityPoiService = communityPoiService; } public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) { this.evaluateFactorService = evaluateFactorService; } + + public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { + this.communityBasePriceService = communityBasePriceService; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java new file mode 100644 index 000000000..bc8c159de --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java @@ -0,0 +1,9 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceService { + + CommunityBasePrice getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java new file mode 100644 index 000000000..049e5ae16 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java @@ -0,0 +1,22 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityBasePriceService; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityBasePriceServiceImpl implements CommunityBasePriceService { + + private CommunityBasePriceDao communityBasePriceDao; + + @Override + public CommunityBasePrice getById(String id) { + return communityBasePriceDao.findById(id); + } + + public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) { + this.communityBasePriceDao = communityBasePriceDao; + } +} From a8cb3041a5b9cda7baa1c0b11485cd27a4355b49 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 24 May 2022 13:28:20 +0800 Subject: [PATCH 10/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E5=9D=87=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 0a7df7866..33ab1b0f7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1280,6 +1280,18 @@ public class PropertyController { houseDtos.add(houseDto); } districtPriceMap.put("houses", houseDtos); + // 新增第三方均价 + try { + double dCredit = 0.61; + Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); + if (null != rMap) { + if (rMap.containsKey("other_price")) { + districtPriceMap.put("other_price", rMap.get("other_price")); + } + } + } catch (Exception ex) { + + } }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); From 089971247554197ae89367d684f4061d58d6fe46 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 24 May 2022 13:53:42 +0800 Subject: [PATCH 11/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E5=9D=87=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 33ab1b0f7..fcf240ddc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -396,6 +396,27 @@ public class PropertyController { put("5000", "five");// 五线 } }; + /** + * 第三方均价映射 + */ + private static final Map OTHER_PRICE_MAP = new HashMap() { + { + put("zhuge_price", "诸葛找房"); + put("anjuke_price", "安居客"); + put("fang_price", "房天下"); + put("lianjia_price", "链家"); + put("cityhouse_price", "禧泰数据"); + put("maitian_price", "麦田"); + put("ke_price", "贝壳"); + put("taofang_price", "淘房365"); + put("zhongyuan_price", "中原"); + put("tbs_price", "兔博士"); + put("qfang_price", "Q房"); + put("fangtan_price", "房探"); + put("fangduoduo_price", "房多多"); + put("wiwj_price", "我爱我家"); + } + }; /** * 利用百度API把GPS转换成Google经纬度 @@ -1255,6 +1276,8 @@ public class PropertyController { } Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8); Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); + Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9); + Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1); List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); if(houses.size() > 3){ districtPriceMap.put("calcType", 1); @@ -1286,7 +1309,18 @@ public class PropertyController { Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); if (null != rMap) { if (rMap.containsKey("other_price")) { - districtPriceMap.put("other_price", rMap.get("other_price")); + Map otherPriceMap = (Map) rMap.get("other_price"); + Map priceResultMap = new HashMap(); + for (String key : otherPriceMap.keySet()) { + Integer price = otherPriceMap.get(key); + // 剔除偏差大于10的 + if (min10Price <= price && max10Price >= price) { + if (OTHER_PRICE_MAP.containsKey(key)) { + priceResultMap.put(OTHER_PRICE_MAP.get(key), price); + } + } + } + districtPriceMap.put("otherPrice", priceResultMap); } } } catch (Exception ex) { From 78b9a5a52220fe452d4e788a4db1718632b264c4 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 24 May 2022 14:30:02 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index fcf240ddc..89d72a2cc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1278,6 +1278,29 @@ public class PropertyController { Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9); Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1); + // 新增第三方均价 + try { + double dCredit = 0.61; + Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); + if (null != rMap) { + if (rMap.containsKey("other_price")) { + Map otherPriceMap = (Map) rMap.get("other_price"); + Map priceResultMap = new HashMap(); + for (String key : otherPriceMap.keySet()) { + Integer price = otherPriceMap.get(key); + // 剔除偏差大于10的 + if (min10Price <= price && max10Price >= price) { + if (OTHER_PRICE_MAP.containsKey(key)) { + priceResultMap.put(OTHER_PRICE_MAP.get(key), price); + } + } + } + districtPriceMap.put("otherPrice", priceResultMap); + } + } + } catch (Exception ex) { + + } List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); if(houses.size() > 3){ districtPriceMap.put("calcType", 1); @@ -1303,29 +1326,6 @@ public class PropertyController { houseDtos.add(houseDto); } districtPriceMap.put("houses", houseDtos); - // 新增第三方均价 - try { - double dCredit = 0.61; - Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); - if (null != rMap) { - if (rMap.containsKey("other_price")) { - Map otherPriceMap = (Map) rMap.get("other_price"); - Map priceResultMap = new HashMap(); - for (String key : otherPriceMap.keySet()) { - Integer price = otherPriceMap.get(key); - // 剔除偏差大于10的 - if (min10Price <= price && max10Price >= price) { - if (OTHER_PRICE_MAP.containsKey(key)) { - priceResultMap.put(OTHER_PRICE_MAP.get(key), price); - } - } - } - districtPriceMap.put("otherPrice", priceResultMap); - } - } - } catch (Exception ex) { - - } }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6);