From 3a5e73267e61bf9d14d3e03784e7756af3808bfe Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Mon, 4 Apr 2022 14:46:55 +0800 Subject: [PATCH 01/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8C=B9=E9=85=8DBUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/service/address/mode/Address.java | 1 + .../service/address/processor/KeywordProcessor.java | 4 ++-- .../address/processor/StageAndPartitionProcessor.java | 10 ++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 1c8f2d004..720da51bb 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -397,6 +397,7 @@ public class Address { } public void setOrgAddress(String orgAddress) { + orgAddress = orgAddress.toUpperCase(); setAddress(orgAddress); if(this.orgAddress != null){ throw new UnsupportedOperationException("原始地址不能被修改"); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java index 547d8d2cb..24ff62e1f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java @@ -70,9 +70,9 @@ public class KeywordProcessor extends AbstractAddressProcessor { if(StringUtils.isNotBlank(address.getAddress())){ if(address.getOrgAddress().indexOf(sb.toString()) >= address.getOrgAddress().indexOf(address.getAddress())){ - sb.insert(0, address.getAddress()); - }else{ sb.append(address.getAddress()); + }else{ + sb.insert(0, address.getAddress()); } String kw = fetchKeywords(address.getAddress()); if(StringUtils.isNotBlank(kw)){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index ab5c929b3..7e4be0d63 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -23,11 +23,17 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { addr = addr.replaceAll(matchResult.matchContent, formatStage); address.setAddress(addr); } - address.setStage(matchResult.matchContent); + address.setStage(formatStage); } matchResult = matchFirst(AddressParserPatterns.PARTITION_PATTERN, orgAddr); if (matchResult != null) { - address.setPartition(matchResult.matchContent); + String formatPartition = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); + if(!formatPartition.equals(matchResult.matchContent)){ + String addr = address.getAddress(); + addr = addr.replaceAll(matchResult.matchContent, formatPartition); + address.setAddress(addr); + } + address.setStage(formatPartition); } return RepeatStatus.NEXT; } From 16798c7d2040d8fe4eb9078a67f86004d00e70db Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Mon, 4 Apr 2022 15:08:34 +0800 Subject: [PATCH 02/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8C=B9=E9=85=8DBUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/address/processor/KeywordProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java index 24ff62e1f..547d8d2cb 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java @@ -70,9 +70,9 @@ public class KeywordProcessor extends AbstractAddressProcessor { if(StringUtils.isNotBlank(address.getAddress())){ if(address.getOrgAddress().indexOf(sb.toString()) >= address.getOrgAddress().indexOf(address.getAddress())){ - sb.append(address.getAddress()); - }else{ sb.insert(0, address.getAddress()); + }else{ + sb.append(address.getAddress()); } String kw = fetchKeywords(address.getAddress()); if(StringUtils.isNotBlank(kw)){ From 0bc39f23428e93e1daedb75ae3f5bacb052e4039 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 6 Apr 2022 11:18:00 +0800 Subject: [PATCH 03/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 122 ++++++++---------- 1 file changed, 53 insertions(+), 69 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 86504ae9a..dcaeb86a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -10,6 +10,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -31,7 +33,7 @@ class SeaweedExport { static void main(String[] args) { def resultMap = new ConcurrentHashMap() - List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + List> data = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\重庆样本匹配结果V4.xlsx") CountDownLatch latch = new CountDownLatch(data.size()) for (List districtT : data) { final List district = districtT @@ -39,72 +41,55 @@ class SeaweedExport { @Override void run() { try { - def city = district.get(1) - def district_code = district.get(2) - def name = district.get(3) - def matchedCity = district.get(22) - def matchedRegion = district.get(23) - def matchedName = district.get(24) - def region; + def no = district.get(0) + def id = district.get(2) + def ret = [] + if (id != null) { + DBObject query = new BasicDBObject() + query.put("_id", new ObjectId(id)) + //query.put("region", matchedRegion) + query.put("cat", "district") + query.put("visible", 0) + //query.put("name", matchedName) - if (map.get("district_code") != null) { - region = map.get("district_code") - } else { - cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> - region = obj.fangjia_district - map.put("district_code", region) + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("developer", 1) + filter.put("estate_company", 1) + filter.put("cubage_percent", 1) // 容积率 + filter.put("virescence_percent", 1) // 绿化率 + filter.put("alias", 1) + filter.put("car_site_count", 1) + filter.put("all_area", 1) + filter.put("loudong_count", 1) + filter.put("household_count", 1) + filter.put("estate_charge", 1) + filter.put("open_date", 1) // 开盘时间 + filter.put("live_date", 1) // 交房时间 + filter.put("elevator", 1) // 电梯 + + DBObject obj = DbUtils.seaweed.findById(id) + if (obj) { + ret[0] = obj.estate_company + ret[1] = obj.developer + ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) + ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) + ret[4] = SeaweedUtils.formatAlias(obj.alias) + ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) + ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) + ret[7] = "" + ret[8] = SeaweedUtils.formatAllArea(obj.all_area) + ret[9] = obj.loudong_count + ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count) + ret[11] = "" + ret[12] = obj.estate_charge + ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) + ret[14] = obj.live_date + ret[15] = obj.is_elevator == true ? "有" : (obj.is_elevator == false ? "无" : "") + ret[16] = no + resultMap.put(no, ret) } } - if (matchedRegion == null) { - matchedRegion = region - matchedName = name - } - def ret = [] - DBObject query = new BasicDBObject() - query.put("city", city) - query.put("region", matchedRegion) - query.put("cat", "district") - query.put("visible", 0) - query.put("name", matchedName) - - DBObject filter = new BasicDBObject() - filter.put("_id", 1) - filter.put("developer", 1) - filter.put("estate_company", 1) - filter.put("cubage_percent", 1) // 容积率 - filter.put("virescence_percent", 1) // 绿化率 - filter.put("alias", 1) - filter.put("car_site_count", 1) - filter.put("all_area", 1) - filter.put("loudong_count", 1) - filter.put("household_count", 1) - filter.put("estate_charge", 1) - filter.put("open_date", 1) // 开盘时间 - filter.put("live_date", 1) // 交房时间 - filter.put("lng", 1) - filter.put("lat", 1) - - DbUtils.seaweed.find(query).limit(1).each { obj -> - ret[0] = obj.estate_company - ret[1] = obj.developer - ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) - ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) - ret[4] = SeaweedUtils.formatAlias(obj.alias) - ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) - ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) - ret[7] = "" - ret[8] = SeaweedUtils.formatAllArea(obj.all_area) - ret[9] = obj.loudong_count - ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count) - ret[11] = "" - ret[12] = obj.estate_charge - ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) - ret[14] = obj.live_date - ret[15] = obj.lng - ret[16] = obj.lat - } - ret[17] = name - resultMap.put(district_code + name, ret) } catch (Exception ex) { ex.printStackTrace() } finally { @@ -115,13 +100,12 @@ class SeaweedExport { }; latch.await(); def ret = []; - ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"]; + ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "是否有电梯", "序号"]; for (List district : data) { - def district_code = district.get(2) - def name = district.get(3) - ret << resultMap.get(district_code + name) + def no = district.get(0) + ret << resultMap.get(no) }; - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息"); + com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\开发商及物业信息.xlsx", ret, "开发商及物业信息"); executorService.shutdown() } } From 3571c5484eb9adf505dc1261e8d45460f0fcbcdd Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 6 Apr 2022 15:37:03 +0800 Subject: [PATCH 04/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy | 4 +++- .../com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index dcaeb86a2..a437ea0c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -29,6 +29,8 @@ class SeaweedExport { static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) static ExecutorService executorService = Executors.newFixedThreadPool(30) + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() static void main(String[] args) { @@ -68,7 +70,7 @@ class SeaweedExport { filter.put("live_date", 1) // 交房时间 filter.put("elevator", 1) // 电梯 - DBObject obj = DbUtils.seaweed.findById(id) + DBObject obj = seaweed.findOne(new ObjectId(id)) if (obj) { ret[0] = obj.estate_company ret[1] = obj.developer diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index f21ff7ae7..668172385 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -83,7 +83,7 @@ class SeaweedUtils { static formatCarSiteCount(def car_site_count) { if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { long val = Long.valueOf(car_site_count) - if (val > 2000 || val < 100) { + if (val > 2500 || val < 100) { return "" } return val; From 4f64858d94f1a9388ecdc7ce085185084663e4d5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 8 Apr 2022 17:21:57 +0800 Subject: [PATCH 05/83] update --- .../fangjia/script/ccbevaluation/Test.groovy | 95 +++-- .../chongqing/BlankEvaluationBatch.groovy | 275 ++++++++++++++ .../script/chongqing/BlankMatch.groovy | 84 +++++ .../script/chongqing/FieldAddValue.groovy | 77 ++++ .../com/fangjia/script/gj/OpenApiUtils.groovy | 2 +- .../gj/VagueAddressLocationBatch.groovy | 2 +- .../gj/data/AddDistrictFieldValue.groovy | 129 ++++--- .../script/gj/data/TempMatchDistrict.groovy | 2 +- .../gj/district/FangjiaClientUtils.groovy | 4 +- .../script/gj/level/DistrictIndexBatch.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 6 +- .../script/gj/paimai/Caa123Crawl.groovy | 338 ++++++++++++------ .../gj/paimai/LianjiaCrawlByChrome.groovy | 9 +- .../script/gj/paimai/PaimaiTest.groovy | 239 +++++++++++-- .../script/gj/price/EvaluationBatch.groovy | 9 +- .../script/loudong/CqfdcChromeCrawl.groovy | 205 +++++++++++ .../loudong/HouseBookChromeCrawl.groovy | 16 +- .../script/loudong/HouseBookCrawl.groovy | 59 +++ .../fangjia/script/loudong/ImageTest.groovy | 147 ++++++++ .../script/thirdPriceCrawl/Esf58Crawl.groovy | 36 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 6 +- .../TestCrawlTotalCount.groovy | 4 +- .../com/fangjia/script/util/DateUtils.groovy | 40 +++ .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 24 files changed, 1531 insertions(+), 259 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 6a9d4446e..9af443b40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -225,17 +225,42 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202202) +//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202203) //ExcelUtils.modify(fileName,{d-> // if(d['C']){ // def oldcount = dealMap.get(d['C']) -// d['V'] = oldcount +// d['X'] = oldcount // def casecount = testMap.get(d['C']) -// d['W'] = casecount +// d['Y'] = casecount // println d // } //},0) +def rows = findTradePpeData(sqlInstance236) +// rows.unique() +def map=findTradeDetail(sqlInstance236,null) +def datas=[] +map.each{key,row-> + if(!rows.contains(key)){ + datas< +// def rows = findTradePpeData(sqlInstance236, 202203,code) +// def alltrades=findTradeDetail(sqlInstance236,code) +// alltrades.each{it-> +// def commid=it[0] +// if(!rows.contains(commid)){ +// datas< // if(d['I'] && d['K']){ @@ -270,24 +295,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) -def index = 0 -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> - def flag = d['M'] - if(flag==0.0){ - if(d['D']){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", new ObjectId(d['D'])) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - index++ - println index - } - } - -},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> +// def flag = d['M'] +// if(flag==0.0){ +// if(d['D']){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", new ObjectId(d['D'])) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// println index +// } +// } +// +//},0) //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ @@ -558,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){ } - - +def findTradePpeData(def sqlInstance){ + def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; + def rows = new HashSet() + sqlInstance.eachRow(sql,{r-> + rows< + def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate + def row=[] + row< + if(row['N'] && row['N'].toString().toDouble()>=20){ + println row['F'] +" "+row['N'] + def dist = CommonDatautils.findDistrictById(row['C']) + for(def weekend : weekends){ + CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0) + } +// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48); + } + },0) + } + + def run1(){ + def city = "重庆" + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row -> + def region = row['E'] + def name = row['F'] + def id = row['C'] + def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30) + def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365) + if (region && name) { + row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name) + + def priceObj = CommonDatautils.findDistrictPrice(city, region, name) + if(priceObj){ + row['AB'] = priceObj.avg_price + } + + def dist = CommonDatautils.findDistrictById(id) + if(dist) { + List list = DbUtils.findAroundDistrict(dist) + def map = getArroundAvgPriceMap(list) + if(map){ + row['L'] = map.avg_price + row['Q'] = map.max_val + row['R'] = map.min_val + } + def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend) + if(lastArroundPrice && priceObj){ + def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice + mod = DataHandleUtils.getDoubleRound(2, mod) + row['P'] = mod + } + def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend) + if(lastYearArroundPrice && priceObj){ + def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice + mod = DataHandleUtils.getDoubleRound(2, mod) + row['O'] = mod + } + } + def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend) + if(lastWeekendObj){ + row['S'] = lastWeekendObj.avg_price + if(priceObj){ + def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price + mod = DataHandleUtils.getDoubleRound(2, mod) + row['M'] = mod + } + } + def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend) + if(lastYearObj){ + row['T'] = lastYearObj.avg_price + if(priceObj){ + def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price + mod = DataHandleUtils.getDoubleRound(2, mod) + row['N'] = mod + } + } + + } + },0) + } + + + def run(){ + def city = "重庆" + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row -> + def region = row['E'] + def name = row['F'] + def id = row['C'] + def area = row['G'].toDouble() + if(region && name){ + def priceObj = CommonDatautils.findDistrictPrice(city, region, name) + if(priceObj){ + row['J'] = priceObj.avg_price + if(row['C']){ + row['I'] = (area*priceObj.avg_price).toInteger() + } + } + def dist = CommonDatautils.findDistrictById(id) + if(dist){ + List list = DbUtils.findAroundDistrict(dist) + row['N'] = getArroundAvgPrice(list) + + def alias = dist.alias + if(!alias){ + alias = [] + } + alias << dist.name + def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10) + if(houseAvgPrice){ + row['K'] = houseAvgPrice + } + + def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10) + row['L'] = tradeAvgPrice + + def avg_price = findAvgPrice(dist._id.toString()) + if(row['J'] && avg_price){ + def weekendPrice = row['J'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['P'] = ppe15 + } + } + } + }); + } + + def findTrades(dist, startDate, startArea, endArea){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + // q.put("source", "链家在线") + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate)) + q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + def price = null + if(prices){ + price = (prices.sum()/prices.size()).toInteger() + } + return price + } + + + def getArroundAvgPriceMap(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + prices.sort() + def map = [:] + map.min_val = prices[0] + map.max_val = prices[prices.size()-1] + def avg_price = (prices.sum()/prices.size()).toInteger() + map.avg_price = avg_price + return map; + } + + def getArroundAvgPrice(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + def avg_price = (prices.sum()/prices.size()).toInteger() + return avg_price; + } + + + def getArroundAvgPriceByWeekend(def list, def weekend){ + def prices = [] + for(def l : list){ + def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend) + if(priceObj){ + prices << priceObj.avg_price + } + } + def avg_price = null; + if(prices){ + avg_price = (prices.sum()/prices.size()).toInteger() + } + return avg_price; + } + + + def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){ + def aliasStr = null; + def total = 0 + def prices = [] + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}" + + ElasticUtils.executeSqlQuery(sql,{ data-> + prices << data.avg_price + }) + }catch(Exception e){ + e.printStackTrace() + } + + def price = null + if(prices){ + price = (prices.sum()/prices.size()).toInteger() + } + return price + } + + public String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy new file mode 100644 index 000000000..75c731a7f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class BlankMatch { + static main(args) { + def task = new BlankMatch() +// task.run() +// task.run1() +// task.run2() + task.run3() + } + + def run3(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") + if(dist){ + row['C'] = dist._id.toString() + } + } + },0) + } + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + for(def d : datas){ + if(d['B'] == row['B']){ + row['C'] = d['C'] + row['D'] = d['D'] + row['E'] = d['E'] + row['F'] = d['J'] + break; + } + } + } + },0) + } + + def run() { + //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); + ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['C'] = jsonObject.get("id") + row['D'] = jsonObject.get("city") + row['E'] = jsonObject.get("district") + row['F'] = jsonObject.get("name") + row['G'] = jsonObject.get("address") + } + } + + + }); + } + + def run1() { + //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row -> + if(row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['C'] = jsonObject.get("id") + row['D'] = jsonObject.get("city") + row['J'] = jsonObject.get("name") + } + + } +// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// row['C'] = jsonObject.get("id") +// row['D'] = jsonObject.get("city") +// row['E'] = jsonObject.get("district") +// row['F'] = jsonObject.get("name") +// row['G'] = jsonObject.get("address") +// } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy new file mode 100644 index 000000000..01ca3f3be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy @@ -0,0 +1,77 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import org.apache.lucene.queryparser.classic.QueryParser +import org.bson.types.ObjectId + +class FieldAddValue { + static main(args) { + def task = new FieldAddValue() + task.run() + } + + def run(){ + //16991 14360 + def districts = CommonDatautils.findAllDistrictByCity("重庆", "district", ["region","name","alias","is_elevator"]) + def index = 0 + def updateIndex = 0 + for(def d : districts){ + if(!d.is_elevator){ + index++ + println index + def alias = d.alias + if(!d.alias){ + alias = [] + } + alias << d.name + def maxHeight = findHouseHeight("重庆", d.region, alias, 20220101) + if(maxHeight && maxHeight>6){ + updateIndex++ + def udata = new BasicDBObject() + udata.put("is_elevator", true) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId( d._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + } + println index+" "+updateIndex + } + + def findHouseHeight(def city, def region, def alias, def startDate){ + def aliasStr = null; + def total = 0 + def heights = null + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') " + + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.height){ + heights = data.height + } + + }) + }catch(Exception e){ + e.printStackTrace() + } + +// def max_height = null +// if(heights){ +// heights.sort() +// max_height = heights[heights.size()-1] +// } + return heights + } + + public String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index ba7287128..244ae41c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils class OpenApiUtils { static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ - address = address.replace("(","(").replace(")",")").replace(" ","") + address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy index b43da6c82..df875945a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy @@ -48,7 +48,7 @@ class VagueAddressLocationBatch extends BaseBatch { void process(Object args) { //"北京","常州", // def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity(); - def citys =["北京"] //CommonDatautils.findAllCity(); + def citys =["重庆"] //CommonDatautils.findAllCity(); //,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄" //市谊大厦 华三新村 阳光家园 for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy index 92e8f1555..7730b81cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy @@ -55,7 +55,7 @@ class AddDistrictFieldValue { // } // }) -// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys; + def citys = ["重庆"]//CcbCommonUtils.citys; //第三方均价数据匹配 // for(def city : citys){ // def rets = findAllDataByCity(city) @@ -63,11 +63,11 @@ class AddDistrictFieldValue { // } //第三方均价数据填补 -// for(def city : citys){ -//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) -// def rMap = getMatchDistrict(city) -// createResultValue(dists, rMap) -// } + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) + def rMap = getMatchDistrict(city) + createResultValue(dists, rMap) + } // for(def city : citys){ // def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"]) @@ -103,52 +103,52 @@ class AddDistrictFieldValue { // } - def citys = ["上海"] - def updateIndex = 0 - def rets = [] - rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"] - for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"]) - def total = dists.size() - - for(def dist : dists){ - if(dist.household_count){ - def ret = [] - ret << dist._id.toString() - ret << dist.city - ret << dist.region - ret << dist.name - ret << dist.household_count - ret << dist.loudong_count - def details = findDistrictDetail(dist) - if(details){ - for(def d : details){ - def region = RegionData.formatRegion(city, d.region) - if(d.region == region){ - if(dist.household_count && dist.household_count==d.house_count){ - ret << d.loudong_count - ret << "链家" -// updateIndex++ -// def udata = new BasicDBObject() -// udata.put("household_count", d.house_count.toInteger()) -// udata.put("m_date", new Date()) -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - } - } - } - }else{ - ret << "" - ret << "" - } - - rets << ret - } - } - ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据") - } +// def citys = ["上海"] +// def updateIndex = 0 +// def rets = [] +// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"] +// for(def city : citys){ +// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"]) +// def total = dists.size() +// +// for(def dist : dists){ +// if(dist.household_count){ +// def ret = [] +// ret << dist._id.toString() +// ret << dist.city +// ret << dist.region +// ret << dist.name +// ret << dist.household_count +// ret << dist.loudong_count +// def details = findDistrictDetail(dist) +// if(details){ +// for(def d : details){ +// def region = RegionData.formatRegion(city, d.region) +// if(d.region == region){ +// if(dist.household_count && dist.household_count==d.house_count){ +// ret << d.loudong_count +// ret << "链家" +//// updateIndex++ +//// def udata = new BasicDBObject() +//// udata.put("household_count", d.house_count.toInteger()) +//// udata.put("m_date", new Date()) +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// } +// } +// } +// }else{ +// ret << "" +// ret << "" +// } +// +// rets << ret +// } +// } +// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据") +// } } @@ -303,6 +303,14 @@ class AddDistrictFieldValue { def createResultValue(dists, rMap){ def updateIndex = 0 def total = dists.size() + def addressIndex = 0 + def allAreaIndex = 0 + def buildAreaIndex = 0 + def virescence_percent_index = 0 + def cubage_percent_index = 0 + def finish_building_date_index = 0 + def household_count_index = 0 + def car_site_count_index = 0 for(def d : dists){ def key = d.city +"#"+d.region+"#"+d.name def sourceMap = rMap.get(key) @@ -311,12 +319,14 @@ class AddDistrictFieldValue { if(!d.address){ def lastValue = getValuesByField(sourceMap,"address", d.estate_type2) if(lastValue){ + addressIndex++ udata.put("address", lastValue) } } if(!d.all_area){ def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2) if(lastValue){ + allAreaIndex++ udata.put("all_area", lastValue) } } @@ -324,6 +334,7 @@ class AddDistrictFieldValue { if(!d.build_area){ def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2) if(lastValue){ + buildAreaIndex++ udata.put("build_area", lastValue) } } @@ -331,6 +342,7 @@ class AddDistrictFieldValue { if(!d.virescence_percent){ def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2) if(lastValue){ + virescence_percent_index ++ udata.put("virescence_percent", lastValue) } } @@ -338,6 +350,7 @@ class AddDistrictFieldValue { if(!d.cubage_percent){ def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2) if(lastValue){ + cubage_percent_index ++ udata.put("cubage_percent", lastValue) } } @@ -345,6 +358,7 @@ class AddDistrictFieldValue { if(!d.finish_building_date){ def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2) if(lastValue){ + finish_building_date_index ++ udata.put("finish_building_date", lastValue) } } @@ -352,6 +366,7 @@ class AddDistrictFieldValue { if(!d.household_count){ def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2) if(lastValue){ + household_count_index ++ udata.put("household_count", lastValue) } } @@ -359,6 +374,7 @@ class AddDistrictFieldValue { if(!d.car_site_count){ def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2) if(lastValue){ + car_site_count_index ++ udata.put("car_site_count", lastValue) } } @@ -373,6 +389,15 @@ class AddDistrictFieldValue { } } } + + println "addressIndex "+ addressIndex + println "allAreaIndex "+ allAreaIndex + println "buildAreaIndex "+ buildAreaIndex + println "virescence_percent_index "+ virescence_percent_index + println "cubage_percent_index "+ cubage_percent_index + println "finish_building_date_index "+ finish_building_date_index + println "household_count_index "+ household_count_index + println "car_site_count_index "+ car_site_count_index } def getValuesByField(sourceMap, field, estateType){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy index 13b8fdde9..a36cf12a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -17,7 +17,7 @@ class TempMatchDistrict { } def stat(){ - def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] + def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"] def total = 0 for(def city : citys){ def count = CommonDatautils.findAllDistrictCount("district", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 6b778545b..44f6d4a9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -38,8 +38,8 @@ class FangjiaClientUtils { } public static def sendDistrictPriceCalculate(def city, def region, def name, def count){ - def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" -// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" +// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" + def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" def content = null; try{ content = new HttpRequest().newGet(url, null, "utf-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 4bd387c7a..e6046e703 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -560,7 +560,7 @@ class DistrictIndexBatch extends BaseBatch{ return datas } - def sortByDistance(d, list){ + def sortByDistance(d, list){ for(def nb : list){ def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) nb.distance = distance diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 6d75b18bd..d54ad9fb3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome { static main(args) { def task = new BeiKeCrawlByChrome(); -// task.run() + task.run() // task.run1() task.addDealRegion(); } @@ -110,7 +110,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(!e.text().contains("南京")){ + if(e.text().contains("福州")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index e2c0d9152..4229926a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.support.ui.ExpectedConditions import org.openqa.selenium.support.ui.WebDriverWait +import javax.swing.JApplet import java.text.DateFormat import java.util.function.Consumer @@ -41,45 +42,137 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca System.setProperty("http.nonProxyHosts", "localhost"); -def doc = getDocument(); -def detailUrls = getDetailLinks(doc); -def rets = batchParse(detailUrls); -saveToDB(rets); + +WebDriver driver = new ChromeDriver() +def proviceMap = getDocument(driver); +def citys = getCity(driver,proviceMap) +def detailUrls = getDetailLinks(driver, citys); -def getDetailLinks(doc){ - def urls = [] - if(doc){ - def eles = JQueryUtils.select(".lots-list .lot-li-a", doc) - if(eles){ - for(def el :eles){ - def href = el.attr("href") - urls << "https://paimai.caa123.org.cn"+href + +def getDetailLinks(driver, cityList){ + + def zxcitys = ["天津市","北京市","上海市","重庆市"] + for(def cityMap : cityList){ + def citys = cityMap.citys + + citys.each{city,id-> + def urls = [] + if(!zxcitys.contains(cityMap.provice)){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + }else{ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + } + + def doc = Jsoup.parse(driver.getPageSource()); + def count = JQueryUtils.selectText(".count", doc) + def totalPage = getTotalPage(count, 12) + if(totalPage){ + for(def i=1;i<=totalPage;i++){ + doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".lots-list .lot-li-a", doc) + if(eles){ + for(def el :eles){ + def href = el.attr("href") + def map = [:] + map.province = cityMap.provice + map.city = city + map.url = "https://paimai.caa123.org.cn"+href + urls << map + } + } + try{ + driver.findElement(By.className("next")).click() + }catch(Exception e){ + Thread.sleep(8000) + driver.findElement(By.className("next")).click() + } + } + println city +" "+urls.size() + def rets = batchParse(urls); + saveToDB(rets); } } + println cityMap.provice +" 完成" + } - return urls; + + //return urls; } -def getDocument(){ - WebDriver driver = new ChromeDriver() +def getTotalPage(def total, def pageSize){ + if(total){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + return 0; +} + +def getCity(driver, proviceMap){ + def rows = [] + def zxcitys = ["天津市","北京市","上海市","重庆市"] + proviceMap.each {k,v-> + if(!zxcitys.contains(k)){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc) + def cityMap = [:] + for(def c : cityEles){ + def cityid = c.attr("data-cityid") + def city = c.text() + if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){ + cityMap.put(city, cityid) + } + + } + def map = [:] + map.provice = k + map.provice_id = v + map.citys = cityMap + rows << map + }else{ + def map = [:] + map.provice = k + map.provice_id = v + def cityMap = [:] + cityMap.put(k, v) + map.citys = cityMap + rows << map + } + } + return rows +} + +def getDocument(driver){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); WebDriverWait wait = new WebDriverWait(driver,20) wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); - def doc = Jsoup.parse(driver.getPageSource()); - - - return doc; + def provinceMap = [:] + if(doc){ + def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc) + for(def ele : proviceEles){ + def provice = ele.text() + def value = ele.attr("data-proviceid") + if(value && provice!="北京市" && provice!="天津市"){ + provinceMap.put(provice, value) + } + } + } + return provinceMap; } def batchParse(def urls){ WebDriver driver = new ChromeDriver() def ret = [] - for(def url :urls){ - driver.get(url); + for(def map :urls){ + driver.get(map.url); def doc = Jsoup.parse(driver.getPageSource()); - def res = parse(url, doc); + def res = parse(map.province, map.city, map.url, doc); if(res){ ret << res } @@ -87,111 +180,126 @@ def batchParse(def urls){ return ret; } -def parse(url,doc){ +def parse(province, city, url,doc){ if(doc){ def result = [:] -// def address = JQueryUtils.selectText(["#pro_bid_name"], doc) -// result.address = address - if(result.address){ - def rMap = OpenApiUtils.matchDistrict(result.address) - if(rMap) { - result.matchCity = rMap.matchCity - result.matchRegion = rMap.matchRegion - result.matchName= rMap.matchName - result.matchAddress = rMap.matchAddress + result.province = province + result.city = city + def address = JQueryUtils.selectText(["#pro_bid_name"], doc) + result.address = address + if(result.address && !result.address.contains("测试")) { + def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num } - } - def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) - if(finishDateStr) { - result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss"); - } + result.status = JQueryUtils.selectText([".bd_state"], doc) + result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc) + result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc) - def curPriceStr = JQueryUtils.selectText([".colorred"], doc) - if(curPriceStr) { - curPriceStr = curPriceStr.replace(",","") - result.tradePrice = curPriceStr.toInteger() - } - - def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc) - if(evaluationPriceStr) { - evaluationPriceStr = evaluationPriceStr.replace(",", "") - if(evaluationPriceStr.isInteger()) { - result.evaluationPrice = evaluationPriceStr.toInteger() + def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) + if (finishDateStr) { + result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); } - } - def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc) - if(depositPriceStr) { - depositPriceStr = depositPriceStr.replace(",", "") - if(depositPriceStr.isInteger()) { - result.depositPrice = depositPriceStr.toInteger() + def curPriceStr = JQueryUtils.selectText([".colorred"], doc) + if (curPriceStr) { + curPriceStr = curPriceStr.replace(",", "") + result.tradePrice = curPriceStr.toDouble().toInteger() } - } - def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc) - if(addPriceStr) { - addPriceStr = addPriceStr.replace(",", "") - if(addPriceStr.isInteger()) { - result.addPrice = addPriceStr.toInteger() - } - } - - def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc) - if(startPriceStr) { - startPriceStr = startPriceStr.replace(",", "").replace("元", "") - if(startPriceStr.isInteger()) { - result.startPrice = startPriceStr.toInteger() - } - } - - result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc) - result.retain = JQueryUtils.selectText(["#retain"], doc) - - def imgs =JQueryUtils.select(".sf-pic-slide img", doc) - if(imgs) { - def imgLinks = [] - for(def img : imgs) { - def href = img.attr("src") - if(href.contains("http")) { - imgLinks << href - }else { - imgLinks << url.substring(0, url.indexOf(":")+1)+href + def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc) + if (evaluationPriceStr) { + evaluationPriceStr = evaluationPriceStr.replace(",", "") + if (evaluationPriceStr.isNumber()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() } } - result.imgUrls = imgLinks - } - def desc = JQueryUtils.selectText(["#J_desc"], doc) - if(!desc){ - desc = JQueryUtils.selectText(["#pro_bid_name"], doc) - } - if(desc) { - if(desc.contains("别墅")) { - result.address = desc.substring(0, desc.indexOf("别墅")+2) - }else if(desc.contains("号楼")) { - result.address = desc.substring(0, desc.indexOf("号楼")+2) - }else if(desc.contains("房屋")) { - result.address = desc.substring(0, desc.indexOf("房屋")+2) + def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc) + if (depositPriceStr) { + depositPriceStr = depositPriceStr.replace(",", "") + if (depositPriceStr.isNumber()) { + result.depositPrice = depositPriceStr.toDouble().toInteger() + } } - } - def descr = JQueryUtils.selectText(["body"], doc) - if(descr) { - def area = getAreaNumber(descr, 5000); - if(area){ - result.area = area.toDouble() + def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc) + if (addPriceStr) { + addPriceStr = addPriceStr.replace(",", "") + if (addPriceStr.isNumber()) { + result.addPrice = addPriceStr.toDouble().toInteger() + } } - } - if(result.area && result.tradePrice){ - result.tradeAvgPrice = (int)(result.tradePrice/result.area) - } - result.content = DataUtil.zip(doc.toString()) - result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() - result.url = url - return result; + def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc) + if (startPriceStr) { + startPriceStr = startPriceStr.replace(",", "").replace("元", "") + if (startPriceStr.isNumber()) { + result.startPrice = startPriceStr.toDouble().toInteger() + } + } + + result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc) + result.retain = JQueryUtils.selectText(["#retain"], doc) + result.lookCount = JQueryUtils.selectText(".look_count span", doc) + + def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc) + if(startDateStr){ + startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() + } + + result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr) + def imgs = JQueryUtils.select(".sf-pic-slide img", doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + } + result.imgUrls = imgLinks + } + + def desc = JQueryUtils.selectText(["#J_desc"], doc) + if (!desc) { + desc = JQueryUtils.selectText(["#pro_bid_name"], doc) + } + if (desc) { + if (desc.contains("别墅")) { + result.address = desc.substring(0, desc.indexOf("别墅") + 2) + } else if (desc.contains("号楼")) { + result.address = desc.substring(0, desc.indexOf("号楼") + 2) + } else if (desc.contains("房屋")) { + result.address = desc.substring(0, desc.indexOf("房屋") + 2) + } + } + + def descr = JQueryUtils.selectText(["body"], doc) + if (descr) { + def area = getAreaNumber(descr, 5000); + if (area) { + result.area = area.toDouble() + } + } + if (result.area && result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / result.area) + } + + result.content = DataUtil.zip(doc.toString()) + result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() + result.url = url + return result; + } } return null } @@ -209,9 +317,11 @@ def saveToDB(rets){ } if(dbo){ - obj.remove("url") + obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) }else{ + obj.c_date = new Date() + obj.m_date = new Date() paimaiResultColl.save(obj) } } @@ -229,6 +339,12 @@ def getAreaNumber(input, maxArea) { def area = null; for (def val : list) { + if(val.toString().contains(" ")){ + val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size()) + } + if(val.contains("室")){ + val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size()) + } def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); if (maxArea) { if (v.toDouble() < maxArea) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 76d63a6a5..f1750cfa9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -417,11 +417,12 @@ public class LianjiaCrawlByChrome { def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] if(cityEles){ - for(def e : cityEles){ + for(def e : cityEles) { def href = e.attr("href") -// if(!href.contains("fang.lianjia") && e.text().contains("福州")){ + if (!href.contains("fang.lianjia") && e.text().contains("福州")) { cityMap.put(e.text(), href) } + } } return cityMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index 44664323e..e35e9f251 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -7,34 +7,147 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver +import java.text.SimpleDateFormat import java.util.regex.Matcher import java.util.regex.Pattern -import java.util.zip.GZIPInputStream -import java.util.zip.GZIPOutputStream -import org.apache.commons.lang.StringUtils; -import java.text.SimpleDateFormat; class PaimaiTest { + + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + static main(args) { def task = new PaimaiTest(); task.run(); } def run(){ - def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134" - def con = find(url); - parser(con, url); + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def url = "https://s.gpai.net/sf/search.do?at=376" + def provinces = getProvice(url, driver) + def cityUrls = getCity(provinces, driver) + getDetailUrls(cityUrls, driver) + } + def getDetailUrls(cityUrls, driver){ + for(def cMap : cityUrls){ + driver.get("https:"+cMap.url) + def doc = Jsoup.parse(driver.getPageSource()); + def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc) - def parser(s, url){ - Document doc = Jsoup.parse(s); + if(total){ + def totalPage = getTotalPage(total.toInteger(), 20) + def detailDatas = [] + for(def i=1;i<=totalPage;i++){ + driver.get("https:"+cMap.url+"&page="+i) + doc = Jsoup.parse(driver.getPageSource()); + def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc) + for(def ele : detailEles){ + def dMap = [:] + dMap.proince = cMap.proince + dMap.city = cMap.city + dMap.url = ele.select("a").attr("href") + detailDatas << dMap + } + } + + def results = parser(detailDatas, driver, cMap.proince, cMap.city) + saveData(results) + println cMap.proince+" "+cMap.city+" "+results.size() + } + } + } + + + def getTotalPage(def total, def pageSize){ + if(total){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + return 0; + } + + def getCity(datas, driver){ + def citys = ["上海","北京","天津","重庆"] + def cityList = [] + for(def data : datas){ + driver.get("https:"+data.href) + def doc = Jsoup.parse(driver.getPageSource()); + if(citys.contains(data.proince)){ + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = data.proince + cityMap.url = data.href + cityList << cityMap + }else{ + def links = JQueryUtils.select(".condition .s-subtab a", doc) + for(def ele : links){ + if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = ele.text() + cityMap.url = ele.attr("href") + cityList << cityMap + } + + } + } + } + return cityList + } + + def getProvice(url, driver){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def datas = [] + def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"] + if(doc){ + def eles = JQueryUtils.select(".condition a", doc) + for(def e : eles){ + def data = [:] + if(!keys.contains(e.text())){ + data.proince = e.text() + data.href = e.attr("href") + datas << data + } + } + } + return datas + } + + def parser(detailDatas, driver, proince, city){ + def results = [] + for(def m : detailDatas){ + try{ + driver.get("https:"+m.url) + Thread.sleep(3000) + def doc = Jsoup.parse(driver.getPageSource()); + def obj = parsering(doc, m.url) + if(obj){ + obj.proince = proince + obj.city = city + results << obj + } + }catch(Exception e){ + println m.url+" 解析有问题!" + } + + } + return results + } + + def parsering(doc, url){ def result = [:] if(doc) { - def title = JQueryUtils.selectText([".d-m-title"], doc) if(!title) { title = JQueryUtils.selectText([".DivItemName"], doc) @@ -48,6 +161,9 @@ class PaimaiTest { } result.title = title result.status = status + if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){ + println title + } if(title) { def newStr =title @@ -76,7 +192,7 @@ class PaimaiTest { if(curPriceStr.contains("成交价:")) { def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","") if(tradePriceStr) { - result.tradePrice = tradePriceStr.toInteger() + result.tradePrice = tradePriceStr.toDouble().toInteger() } } } @@ -85,23 +201,28 @@ class PaimaiTest { curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc) if(curPriceStr) { curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","") - result.tradePrice = curPriceStr.toInteger() + result.tradePrice = curPriceStr.toDouble().toInteger() } } - def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc) - if(enrollCount) { + def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc) + if(enrollCount && enrollCount.isNumber()) { result.enrollCount = enrollCount.toInteger() } - def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc) - if(callCount) { + def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc) + if(callCount && callCount.isNumber()) { result.callCount = callCount.toInteger() } - def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc) - if(lookCount) { - result.lookCount = lookCount.toInteger() + def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc) + if(lookCount && lookCount.isNumber()) { + try{ + result.lookCount = lookCount.toInteger() + }catch(Exception e){ + println lookCount + } + } def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc) @@ -140,7 +261,7 @@ class PaimaiTest { }else if(str.contains("保证金")) { def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "") if(v) { - result.depositPrice = v.toInteger() + result.depositPrice = v.toDouble().toInteger() } }else if(str.contains("延时周期")) { @@ -184,6 +305,47 @@ class PaimaiTest { } } } + + if(detailMap && !detailMap.get("拍品名称")){ + detailMap = [:] + for(def tr : trs) { + + if(tr.text().contains("拍品名称")){ + def td1 = tr.select("td:contains(拍品名称)+td").text() + detailMap.put("拍品名称", td1) + } + + if(tr.text().contains("权利来源")){ + def td1 = tr.select("td:contains(权利来源)+td").text() + detailMap.put("权利来源", td1) + } + + if(tr.text().contains("权证情况")){ + def td1 = tr.select("td:contains(权证情况)+td").text() + detailMap.put("权证情况", td1) + } + + if(tr.text().contains("拍品所有人")){ + def td1 = tr.select("td:contains(拍品所有人)+td").text() + detailMap.put("拍品所有人", td1) + } + + if(tr.text().contains("房屋情况")){ + def td1 = tr.select("td:contains(房屋情况)+td").text() + detailMap.put("房屋情况", td1) + } + + if(tr.text().contains("权利限制情况及瑕疵情况")){ + def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text() + detailMap.put("权利限制情况及瑕疵情况", td1) + } + + if(tr.text().contains("拍品介绍")){ + def td1 = tr.select("td:contains(拍品介绍)").text() + detailMap.put("拍品介绍", td1) + } + } + } result.detailMap = detailMap if(!result.address) { result.address = detailMap.get("拍品名称") @@ -248,17 +410,18 @@ class PaimaiTest { } } - result.content = DataUtil.zip(s) +// result.content = DataUtil.zip(s) def el = JQueryUtils.select([".d-block table:eq(1)"], doc) if(!el) { el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc) } - result.content = DataUtil.zip(s) +// result.content = DataUtil.zip(s) result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() result.url = url result.mDate = new Date(); + return result; } def find(url){ @@ -374,4 +537,32 @@ class PaimaiTest { } return null; } + + def saveData(def results){ + + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy index 40b17049e..084a3d183 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy @@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +class BlankEvaluationBatch { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -class EvaluationBatch { static main(args) { - def task = new EvaluationBatch(); + def task = new BlankEvaluationBatch(); task.run() } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy new file mode 100644 index 000000000..e5ffbd54f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy @@ -0,0 +1,205 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class CqfdcChromeCrawl { + + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) + + + static main(args) { + def task = new CqfdcChromeCrawl(); + task.run() + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def list = getListDatas(driver) + getDetails(driver, list); + + } + + + + def getDetails(driver, list){ + def index = 0 + for(def map_old : list){ + def datas = [] + index++ + def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}" +// driver.get(url) +// def doc = driver.getPageSource() + def cityDoc = getDocment(url, driver) + def trs = JQueryUtils.select(".building_table tr", cityDoc) + def unit1 = null + def unit2 = null + def unit3 = null + def unit1_text = null + def unit2_text = null + def unit3_text = null + for(def i=0;i + map.put(k,v) + } + datas << map + } + } + } + } + saveDatas(datas) + println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size() + } + } + + def getDocment(def url, def driver){ + def stop = false + def doc = null + def error = 0 + while (!stop || error>=10){ + try{ + driver.get(url) + def docment = driver.getPageSource() + if(docment){ + doc = Jsoup.parse(docment) + stop = true; + }else{ + error ++ + } + }catch(Exception e){ + error ++ + } + } + return doc; + } + + def getListDatas(driver){ + driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html") + def datas = [] + for(def i=2;i<=208;i++){ + def docment = Jsoup.parse(driver.getPageSource()); + def trs = JQueryUtils.select("#searchbody tr", docment) + for(def tr : trs){ + def tds = tr.select("td") + + def hrefs = tds.get(3).select("a") + for(def href : hrefs){ + def map = [:] + map.house_num_desc = href.text() + def str = href.attr("href") + def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','")) + map.building_id = building_id + + map.city = "重庆" + map.name = tds.get(0).text() + map.company = tds.get(1).text() + map.license_key = tds.get(2).text() + datas << map + } + } + println i +" "+datas.size() + try{ + driver.findElement(By.id("nextpage")).click() + }catch(Exception e){ + Thread.sleep(8000) + driver.findElement(By.id("nextpage")).click() + } + + } + println "列表:"+datas.size() + return datas; + } + + + def saveDatas(def datas){ + def rows = [] + def keys = [] + for(def d : datas){ + def q = new BasicDBObject() + q.put("city", d.city) + q.put("name",d.name) + q.put("house_num_desc", d.house_num_desc) + q.put("room_num", d.room_num) + q.put("unit_num",d.unit_num) + def obj = loudongConn.findOne(q) + if(!obj){ + def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num + if(!keys.contains(key)){ + def udata = new BasicDBObject() + d.each {k,v-> + udata.put(k,v) + } + udata.put("c_date", new Date()) + rows << udata + keys << key + } + } + } + if(rows){ + loudongConn.insert(rows); + println "save: "+rows.size() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy index 31ab75ba6..8684414c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy @@ -30,8 +30,8 @@ class HouseBookChromeCrawl { WebDriver driver = new ChromeDriver() def listPageUrl = getIndexPage(driver) def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl) - keywords.remove("城南家园") - keywords.remove("康居西城") +// keywords.remove("城南家园") +// keywords.remove("康居西城") println city +"共 "+keywords.size()+" 个小区" getDistrictIndex(driver, keywords) @@ -61,15 +61,15 @@ class HouseBookChromeCrawl { def getAllDist(driver, url, page){ def keywords = [] - for(def i=1;i<=page;i++){ + for(def i=73;i<=page;i++){ driver.get(url+"?selectPages="+i) def doc = Jsoup.parse(driver.getPageSource()) def eles = JQueryUtils.select(".title-area .title-text", doc) for(def e : eles){ def text = e.text() - if(text=="富州北郡"){ - println i - } +// if(text=="富州北郡"){ +// println i +// } keywords << text } } @@ -206,7 +206,7 @@ class HouseBookChromeCrawl { detail.estate_type = house_type } if(detail){ - =-09ql.floor = floor + detail.floor = floor rows << detail } } @@ -214,7 +214,7 @@ class HouseBookChromeCrawl { } } return rows; - }+*-* + } def getIndexPage(driver){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy new file mode 100644 index 000000000..8a9e81f31 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.thirdPriceCrawl.UserAgents +import com.fangjia.script.util.ABuYunUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.jsoup.Jsoup +import org.jsoup.select.Elements + +class HouseBookCrawl { + + public def CHARSET = "utf-8"; + + static main(args) { + def task = new HouseBookCrawl() + task.run() + } + +// def run(){ +// goTocitypage() +// } +// def goTocitypage(){ +// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html" +// def headers = [:] +// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); +// headers.put("User-Agent", UserAgents.get()) +// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582") +// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e") +// def params = [:] +// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false) +// def cookie = getCookie(responseMap) +// } +// +// def getCookie(responseMap) { +// def cookieAuth = null +// try { +// def response = responseMap.get("response"); +// if (StringUtils.isNotEmpty(response)) { +// JSONObject reJson = JSONObject.fromObject(response) +// //登录成功获取COOKIE +// if (reJson.success) { +// responseMap.get("headerFields").each { k, v -> +// if (k && k.equals("Set-Cookie")) { +// if (v) { +// cookieAuth = v.join(';') +// } +// } +// } +// } +// } +// } catch (Exception e) { +// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage() +// } +// println("最终构造cookieAuth为:" + cookieAuth) +// return cookieAuth +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy new file mode 100644 index 000000000..1e7978919 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.loudong + +import net.sourceforge.tess4j.ITesseract +import net.sourceforge.tess4j.Tesseract +import net.sourceforge.tess4j.util.ImageHelper +import org.apache.commons.io.FileUtils +import org.ehcache.xml.model.TimeUnit +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.OutputType +import org.openqa.selenium.Point +import org.openqa.selenium.TakesScreenshot +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver + +import javax.imageio.ImageIO +import java.awt.image.BufferedImage + +public class ImageTest { + + public static void main(String[] args) throws IOException { +// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif"); +// Tesseract instance = new Tesseract(); +// + +// //将验证码图片的内容识别为字符串 +// String result = instance.doOCR(imageFile); +// println result + + //1.获取网页的浏览器driver + //(1)定义url + String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#"; + //(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径) + String path = System.getProperty("user.dir"); + System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + //(3)建立selenium 驱动 + //WebDriver driver = new ChromeDriver(options); + WebDriver driver = new ChromeDriver(); + //(4)设置窗口最大化及网页等待时间 + driver.manage().window().maximize(); //窗口最大化 +// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS); + //(5)启动driver浏览网页 + driver.get(url); //输入网址 + Thread.sleep(1000); + + driver.findElement(By.id("cell1-11-2")).click() + Thread.sleep(1000); + + def docment = Jsoup.parse(driver.getPageSource()); + println docment + + WebElement ele = driver.findElement(By.id("vimg")); + + File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE); + BufferedImage pic1 = ImageIO.read(screenshot); + Point point = ele.getLocation(); + + //(2)输入验证码 + File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); + FileUtils.copyFile(srcfile, new File("picture/source.png")); + //原图片 + //String soursePicPath = "D:\\data_dev\\test\\test.png"; + String soursePicPath = "picture/source.png"; + File sourcePic = new File(soursePicPath); + int eleWid = ele.getSize().getWidth() + int eleHeiht = ele.getSize().getHeight() + + BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht); + ImageIO.write(pic2, "jpg", sourcePic); + + //File desImage = new File("D:\\data_dev\\test\\test02.png"); + File desImage = new File("picture/answer.png"); + + + + //加载待读取图片 + //File imageFile = new File("D:\\data_dev\\test\\test02.png"); + File imageFile = new File("picture/answer.png"); + BufferedImage image = ImageIO.read(imageFile); + //对图片进行处理 + image = convertImage(image); + //创建tess对象 + ITesseract instance = new Tesseract(); + //设置训练文件目录 + //instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata"); + String path02 = System.getProperty("user.dir"); + instance.setDatapath(path02 + "\\tessdata"); + //设置训练语言 + //instance.setLanguage("chi_sim"); + instance.setLanguage("eng"); + //执行转换 + String s = instance.doOCR(image); + + System.out.println(s); + String[] split = s.replace(" ", "").split("=|:"); + String a = split[split.length - 1].trim(); + char[] chars = a.toCharArray(); + if (chars.length == 2) { + if (chars[0] == '7') { + chars[0] = '1'; + } + } + a = String.valueOf(chars); + + System.out.println(a); + String b = s.substring(0, 1); + System.out.println(b); + String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim())); + + + WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]")); + input2.sendKeys(answer); + Thread.sleep(4000); + //(3)点击"查询"按钮 + driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click(); + + String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄 + for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历 + if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出 + driver.close(); + driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver + System.out.println(driver.getCurrentUrl()); //打印是否为新窗口 + } + } + + Thread.sleep(4000); + + //4.关闭浏览器 + driver.close(); + + } + + + //对图片进行处理 - 提高识别度 + public static BufferedImage convertImage(BufferedImage image) throws Exception { + //按指定宽高创建一个图像副本 + //image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight()); + //图像转换成灰度的简单方法 - 黑白处理 + image = ImageHelper.convertImageToGrayscale(image); + //图像缩放 - 放大n倍图像 + image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15); + return image; + } + + +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d7d81e90e..9362389ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -191,24 +191,24 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] - for(def city : citys){ - def str = cityList.get(city) - for(def i=0;i<1;i++){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } - } - } +// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] +// for(def city : citys){ +// def str = cityList.get(city) +// for(def i=0;i<1;i++){ +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// } +// } +// } - /*cityList.each{city, str-> + cityList.each{city, str-> for(def i=0;i<1;i++){ - if(citys.contains(city)){ +// if(citys.contains(city)){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); if(districts){ def arrs = str.split("\\|") @@ -218,9 +218,9 @@ class Esf58Crawl extends BaseBatch{ addCrawlKeys(city); addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); } - } +// } } - }*/ + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 1ce2d438f..0ca7c1afb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,10 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", -// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] + def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", + "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] //"fang.com":"搜房", - def sourceMap = ["fang.com":"搜房"] +// def sourceMap = ["fang.com":"搜房"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index db48455df..bd0fbb3f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202201); - def count2 = findTotalCount(source, 202202); + def count1 = findTotalCount(source, 202202); + def count2 = findTotalCount(source, 202203); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index bfa7f8ec7..f6399a2dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.util +import org.apache.commons.lang.StringUtils + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; @@ -995,4 +997,42 @@ class DateUtils { return rtn; } + public static Date parseString2Date(String str){ + if(StringUtils.isBlank(str)){ + return null + } + str = str.replaceAll("^\\s+", "").replaceAll("\\s+\$", "") + + Date now = new Date() + if(str.contains("小时")||str.contains("分钟") || str.contains("秒") || str.contains("当前")|| str.contains("今天")){ + return org.apache.commons.lang3.time.DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH) + }else if(str.contains("昨天")) { + return add(now, -1) + }else if(str.contains("天")){ + int days = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*days) + }else if(str.contains("月前")){ + int month = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*month*30) + }else if(str.contains("年前")){ + int years = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*years*365) + }else{ + if(str.size()<10){ + if(str.contains("-")){ + str = (now.getYear()+1900)+"-"+str + } + }else{ + if(str.contains("月") && str.contains("日") && !str.contains("年")){ + str = (now.getYear()+1900)+"年"+str.substring(0, str.indexOf("日")+1) + } + } + if(str.contains("年")){ + return DateUtils.parseFromTightByPattern(str, "yyyy年MM月dd日") + }else{ + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + } + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index e0d8713b6..3810aa9f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220201", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220301", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220301", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220401", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 From 2a8dfcedd65218a0213899d2a68e851cf1c154f4 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 18 Apr 2022 10:00:52 +0800 Subject: [PATCH 06/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index a437ea0c3..9817c00cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -76,7 +76,7 @@ class SeaweedExport { ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) - ret[4] = SeaweedUtils.formatAlias(obj.alias) + ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias) ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) ret[7] = "" From 90de7d5288d317458c93f142e8da92d74843bade Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 18 Apr 2022 15:08:57 +0800 Subject: [PATCH 07/83] =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 2749ed7f4..4978bd962 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) From 54c92a53cc69c2c63b7d285af63ebb15fd4e1be9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 18 Apr 2022 15:17:15 +0800 Subject: [PATCH 08/83] update --- projects/fangjia-script/pom.xml | 27 + .../ccbevaluation/BigdataDealBatch.groovy | 1 + .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../com/fangjia/script/gj/OpenApiUtils.groovy | 2 +- .../script/gj/paimai/Caa123Crawl.groovy | 63 ++- .../fangjia/script/gj/paimai/DataUtil.groovy | 290 ++++++++++ .../script/gj/paimai/JdPaimaiCrawl.groovy | 528 ++++++++++++++++++ .../script/gj/paimai/PaimaiDataUtil.groovy | 79 ++- .../script/gj/paimai/PaimaiTest.groovy | 53 +- .../script/gj/paimai/RmfysszcCrawl.groovy | 419 +++++++++++--- .../fangjia/script/loudong/CqfdcMatch.groovy | 173 ++++++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 83 +-- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 6 +- 15 files changed, 1564 insertions(+), 172 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index c71006bb8..9469b6d02 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -494,5 +494,32 @@ msgpack 0.6.12 + + + + com.asprise.ocr + java-ocr-api + [15,) + + + + net.java.dev.jna + jna + 4.1.0 + + + + net.sourceforge.tess4j + tess4j + 3.4.0 + + + com.sun.jna + jna + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index aa12539b9..d34dedcc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{ def q = new BasicDBObject() q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate)); q.put("area", new BasicDBObject('$gte', 20)) + q.put("status", 0) def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 412677ae5..2476de4c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index a8a616b7c..5a6b21aac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 244ae41c0..77239e3ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils class OpenApiUtils { static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ - address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","") + address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index 4229926a1..afaae338f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -88,9 +88,10 @@ def getDetailLinks(driver, cityList){ driver.findElement(By.className("next")).click() } } - println city +" "+urls.size() + def rets = batchParse(urls); saveToDB(rets); + println city +" "+urls.size() } } println cityMap.provice +" 完成" @@ -123,7 +124,7 @@ def getCity(driver, proviceMap){ for(def c : cityEles){ def cityid = c.attr("data-cityid") def city = c.text() - if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){ + if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ cityMap.put(city, cityid) } @@ -170,8 +171,7 @@ def batchParse(def urls){ WebDriver driver = new ChromeDriver() def ret = [] for(def map :urls){ - driver.get(map.url); - def doc = Jsoup.parse(driver.getPageSource()); + def doc = getDocument(map.url, driver) def res = parse(map.province, map.city, map.url, doc); if(res){ ret << res @@ -180,6 +180,30 @@ def batchParse(def urls){ return ret; } +def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc +} + def parse(province, city, url,doc){ if(doc){ def result = [:] @@ -205,7 +229,12 @@ def parse(province, city, url,doc){ def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) if (finishDateStr) { - result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); + try{ + result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); + }catch(Exception e){ + + } + } def curPriceStr = JQueryUtils.selectText([".colorred"], doc) @@ -255,7 +284,12 @@ def parse(province, city, url,doc){ startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() } - result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr) + try{ + result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim()) + }catch(Exception e){ + + } + def imgs = JQueryUtils.select(".sf-pic-slide img", doc) if (imgs) { def imgLinks = [] @@ -315,7 +349,7 @@ def saveToDB(rets){ ret.each{k,v-> obj.put(k,v) } - + obj.source= "中拍网" if(dbo){ obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) @@ -346,15 +380,18 @@ def getAreaNumber(input, maxArea) { val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size()) } def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); - if (maxArea) { - if (v.toDouble() < maxArea) { - area = v; + if(v && v.toString().isNumber()){ + if (maxArea) { + if (v.toDouble() < maxArea) { + area = v; + break; + } + } else { + area = v break; } - } else { - area = v - break; } + } if (area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy new file mode 100644 index 000000000..8cce171c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -0,0 +1,290 @@ +package com.fangjia.script.gj.paimai + +import org.apache.commons.codec.binary.Base64 +import org.apache.commons.lang.StringUtils +import sun.font.Font2D +import sun.font.Font2DHandle +import sun.font.TrueTypeFont +import sun.font.TrueTypeGlyphMapper + +import java.awt.Font +import java.awt.font.FontRenderContext +import java.awt.font.GlyphVector +import java.awt.geom.AffineTransform +import java.lang.reflect.Field +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataUtil { + static def getAreaNumber(def input, def maxArea) { + Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)') + Matcher matcher = pattern.matcher(input) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() > 0) { + def area = null + + for (def val : list) { + def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "") + if (maxArea) { + if (v.isDouble()) { + if (v.toDouble() < maxArea) { + area = v + break + } + } + } else { + area = v + break + } + } + + if (area) { + return area + } + } + return null + } + + static def getPaiMaiType(def s) { + if (s) { + if (s.toString().contains("结束")) { + return "已完成" + } else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) { + return "进行中" + } else if (s.toString().contains("即将开始")) { + return "即将开始" + } else if (s.toString().contains("流拍")) { + return "流拍" + } else if (s.toString().contains("撤回")) { + return "撤回" + } else if (s.toString().contains("暂缓")) { + return "暂缓" + } else if (s.toString().contains("中止")) { + return "中止" + } else if (s.toString().contains("失败")) { + return "失败" + } + } + return s + } + + + static def getDateRange(def s) { + s = s.replace("起", "") + Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + def str = matcher.group() + list.add(str) +// if(str) { +// def indx1 = str.indexOf("年") +// def indx2 = str.indexOf("月") +// def month = str.substring(indx1+1, indx2); +// if(month.toInteger()<12) { +// list.add(str); +// } +// } + } + if (list.size() >= 1) { + String s1 = list.get(0) + s1 = s1.replace("法院将于", "").replace(" ", "") + def array = s1.split("至") + return array + } + return null + } + + + static def getStartPrice(def s) { + Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() >= 1) { + String s1 = list.get(0) + if (s1.contains("万元")) { + s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "") + return (int) (s1.toDouble() * 10000) + } else { + s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "") + if (s1.isInteger()) { + return s1.toInteger() + } else { + return (int) (s1.toDouble()) + } + } + } + return null + } + + static def getEstateType(def s) { + Pattern pattern = Pattern.compile('房屋类型:.*;') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() >= 1) { + String s1 = list.get(0) + if (s1) { + s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";")) + return s1.replace(")", "").replace("】", "") + } + } + return null + } + + static def findEstateType(def s) { + if (s) { + def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"] + for (def type : villTypes) { + if (s.contains(type)) { + return "别墅" + } + } + + def districtTypes = ["住宅", "公寓"] + for (def type : districtTypes) { + if (s.contains(type)) { + return "普通住宅" + } + } + } + + return null + + } + + static def filterChar(def detailMap) { + def newDetailMap = [:] + detailMap.each { def k, v -> + if (k.toString().contains(".")) { + newDetailMap.put(k.toString().replace(".", "\\u002e"), v) + } else { + newDetailMap.put(k, v) + } + } + return newDetailMap + } + + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + /** + * font-secret字符串专用解密工具 + * + * @param key 密匙 + * @param encodeString 加密后的字符串 + * @return 解密后的字符串 + */ + static String decodeString(String key, String encodeString) { + if (!key) { + return "" + } + try { + //base64解码,初始化字体 + Base64 base64 = new Base64() + byte[] ss = base64.decodeBase64(key) + InputStream inputStream = new ByteArrayInputStream(ss) + Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream) + FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false) + GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "") + + //获取font中字形的映射关系,字段为private,使用反射 + Class clazz = Font.class + Field[] fs = clazz.getDeclaredFields() + Font2DHandle font2DHandle = null + for (int i = 0; i < fs.length; i++) { + fs[i].setAccessible(true)// 将目标属性设置为可以访问 + if (fs[i].getName().equals("font2DHandle")) { + font2DHandle = (Font2DHandle) fs[i].get(dynamicFont) + } + + } + + //得到映射关系 + Font2D font2D = font2DHandle.font2D + TrueTypeFont trueTypeFont = (TrueTypeFont) font2D + TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper() + + //开始解密,encodeString为加密后的字符串 + StringBuffer buffer = new StringBuffer() + char[] chars = encodeString.toCharArray() + for (int i = 0; i < chars.length; i++) { + buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1) + } + return buffer.toString() + } catch (Exception e) { + e.printStackTrace() + } + return "" + } + + static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"] + + static String urlToSiteName(String url) { + if (StringUtils.isEmpty(url)) { + return null + } + + for (String suffix : SITE_SUFFIXS) { + if (url.indexOf(suffix) > -1) { + String s = StringUtils.substringBefore(url, suffix) + if (s.indexOf(".") > -1) { + s = StringUtils.substringAfterLast(s, ".") + if (StringUtils.isNotBlank(s)) { + return s + suffix + } + } else if (s.indexOf("://") > -1) { + s = StringUtils.substringAfterLast(s, "://") + if (StringUtils.isNotBlank(s)) { + return s + suffix + } + } else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) { + return s + suffix + } + + return null + } + } + return null + } + + static double getRound(int scale, double dSource) { + BigDecimal deSource = new BigDecimal(dSource) + return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue() + } + + static def filterEmpty(def s){ + if(s && s.contains("暂无")){ + return null + } + return s + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy new file mode 100644 index 000000000..aac9c7315 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -0,0 +1,528 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +import javax.sound.midi.MidiSystem +import java.util.regex.Pattern + +class JdPaimaiCrawl { + + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new JdPaimaiCrawl(); + task.run(); + } + def run(){ + def existsIds = findAllIds() + println "已存在:"+existsIds.size() + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def links = getList(driver) + def ids = getDetailLinks(links, driver) + println "共:"+ids.size() + parserList( existsIds,ids, driver) + + + + } + + def getList(driver){ + def list = [] + for(def i=1;i<=32;i++){ + if(i==8){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" + }else if(i==5){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" + }else if(i==17){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" + }else if(i==19){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" + }else{ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" + } + } + def links = [] + for(def url : list){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + //def document = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(doc){ + def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")")) + JSONObject obj = JSONObject.fromObject(json); + if(obj && obj.total){ + def total = obj.total + def page = total/40 + def p = page.toInteger() + if(page > p){ + p = p +1 + } + + for(def i=1;i<=p;i++){ + links << url+"&page="+i + } +// if(links){ +// break; +// } + } + } + } + return links; + } + + def getDetailLinks(links, driver){ + def rets = [] + def citys = ["北京","上海","天津","重庆"] + for(def link : links){ + driver.get(link) + def doc = Jsoup.parse(driver.getPageSource()); + if(doc) { + def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")")) + JSONObject obj = JSONObject.fromObject(json); + if (obj && obj.ls) { + for(def o : obj.ls){ + def map = [:] + if(citys.contains(o.province)){ + map.province = o.province + map.city = o.province + map.region = o.city + }else{ + map.province = o.province + map.city = o.city + } + map.id = o.id.toString() + + rets << map + } + } + } + } + return rets + } + + def parserList(existsIds,list, driver){ + def rets = [] + for(def map : list) { + if(!existsIds.contains(map.id)){ + def url = "https://paimai.jd.com/" + map.id + def doc = getDocument(url, driver) + if (doc) { + def result = parsering(doc, url, map.province, map.city, map.region) + if (result) { + rets << result + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + + } + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } + + def parsering(doc, def url, def province, def city, def region){ + def result = [:] + if(doc) { + result.proince = province + result.city = city + if(region){ + result.region = region + } + def title = JQueryUtils.selectText([".pm-name"], doc) + result.title = title + result.address = JQueryUtils.selectText([".pm-location em"], doc) + if(result.address) { + //调用获取城市 +// if(!city) { +// city = RegionData.formatCity(cityMap.name) +// } + + if(city) { + def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + + } + + result.status = JQueryUtils.selectText([".pm-summary .state"], doc) + + def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + + def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "") + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc) + if(startPriceStr) { + startPriceStr = startPriceStr.replace(",", "") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + + } + + def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc) + if(tradePriceStr) { + tradePriceStr = tradePriceStr.replace(",", "") + if(tradePriceStr.isInteger()) { + result.tradePrice = tradePriceStr.toInteger() + }else { + result.tradePrice = (int)(tradePriceStr.toDouble()) + } + + } + + def finishDateStr = JQueryUtils.selectText([".endtime"], doc) + if(finishDateStr && finishDateStr.contains("结束时间")) { + finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" ")) + result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss") + } + + def lis = JQueryUtils.select(".description ul li", doc) + if(lis && lis.size()) { + for(def li : lis) { + def str = li.select("span").text() + if(str.contains("起拍")) { + def v= li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + if(v.isInteger()) { + result.startPrice = v.toDouble().toInteger() + }else { + result.startPrice = (int)(v.toDouble()) + } + + } + }else if(str.contains("评估价")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v.isInteger()) { + result.evaluationPrice = v.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(v.toDouble()) + } + }else if(str.contains("加价幅度")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + result.addPrice = v.toDouble().toInteger() + } + }else if(str.contains("保证金")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + + } + }else if(str.contains("延时周期")) { + + result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "") + + }else if(str.contains("优先竞买权人")) { + result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "") + } + } + } + + def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + }else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + }else { + if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + + def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("href") + if(href.contains("http")) { + imgLinks << href + }else { + imgLinks << url.substring(0, url.indexOf(":")+1)+href + } + } + result.imgUrls = imgLinks + } + + } + + //解析面积 + def article = JQueryUtils.selectText([".floor-content"], doc) + if(article) { + def area = DataUtil.getAreaNumber(article, 5000) + if(!area) { + article = JQueryUtils.selectText(["body"], doc) + if(article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = DataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + + for(def j=0;jind+1) { + date = date +"时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if(j==0) { + try{ + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + }catch(Exception e){ + + } + + }else { + if(!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + if(!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) + } + + def estateType = DataUtil.findEstateType(s1) + if(estateType) { + result.matchEstateType = estateType + } + + } + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString() + result.url = url + result.city = city + result.mDate = new Date() + result.source = "京东" + return result; + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } + def findAllIds(){ + def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) + def dbcursor = paimaiResultColl.find(q) + def ids = [] + for(def d : dbcursor){ + def url = d.url + if(url){ + def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length()) + ids << id + } + } + return ids; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy index f98faa3ee..90a450a66 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.paimai +import java.text.SimpleDateFormat import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream @@ -94,32 +95,32 @@ class PaimaiDataUtil { } - public static def getDateRange(def s) { - s = s.replace("起","") - Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)'); - Matcher matcher = pattern.matcher(s); - - List list = new ArrayList(); - while (matcher.find()) { - def str = matcher.group() - list.add(str); -// if(str) { -// def indx1 = str.indexOf("年") -// def indx2 = str.indexOf("月") -// def month = str.substring(indx1+1, indx2); -// if(month.toInteger()<12) { -// list.add(str); -// } -// } - } - if(list.size()>=1) { - String s1 = list.get(0) - s1 = s1.replace("法院将于", "").replace(" ","") - def array = s1.split("至"); - return array - } - return null; - } +// public static def getDateRange(def s) { +// s = s.replace("起","") +// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)'); +// Matcher matcher = pattern.matcher(s); +// +// List list = new ArrayList(); +// while (matcher.find()) { +// def str = matcher.group() +// list.add(str); +//// if(str) { +//// def indx1 = str.indexOf("年") +//// def indx2 = str.indexOf("月") +//// def month = str.substring(indx1+1, indx2); +//// if(month.toInteger()<12) { +//// list.add(str); +//// } +//// } +// } +// if(list.size()>=1) { +// String s1 = list.get(0) +// s1 = s1.replace("法院将于", "").replace(" ","") +// def array = s1.split("至"); +// return array +// } +// return null; +// } public static def getStartPrice(def s) { @@ -159,4 +160,30 @@ class PaimaiDataUtil { return newDetailMap; } + public static def getDateRange(def s) { + s = s.replace("起","") + Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)'); + Matcher matcher = pattern.matcher(s); + + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>=1) { + String s1 = list.get(0) + s1 = s1.replace("法院将于", "") + def array = s1.split("至"); + return array + } + return null; + } + + public static Date parseFromTightByPattern(String s, String pattern){ + if (s == null || s.length() == 0){ + return null + } + SimpleDateFormat format = new SimpleDateFormat(pattern); + return format.parse(s) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index e35e9f251..acced0959 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -83,15 +83,19 @@ class PaimaiTest { driver.get("https:"+data.href) def doc = Jsoup.parse(driver.getPageSource()); if(citys.contains(data.proince)){ - def cityMap = [:] - cityMap.proince = data.proince - cityMap.city = data.proince - cityMap.url = data.href - cityList << cityMap + def links = JQueryUtils.select(".condition .s-subtab a", doc) + for(def ele : links) { + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = data.proince + cityMap.region =ele.text() + cityMap.url = data.href + cityList << cityMap + } }else{ def links = JQueryUtils.select(".condition .s-subtab a", doc) for(def ele : links){ - if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ + if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ def cityMap = [:] cityMap.proince = data.proince cityMap.city = ele.text() @@ -109,7 +113,7 @@ class PaimaiTest { driver.get(url) def doc = Jsoup.parse(driver.getPageSource()); def datas = [] - def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"] + def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"] if(doc){ def eles = JQueryUtils.select(".condition a", doc) for(def e : eles){ @@ -123,11 +127,34 @@ class PaimaiTest { } return datas } + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } def parser(detailDatas, driver, proince, city){ def results = [] for(def m : detailDatas){ - try{ + // try{ driver.get("https:"+m.url) Thread.sleep(3000) def doc = Jsoup.parse(driver.getPageSource()); @@ -137,9 +164,9 @@ class PaimaiTest { obj.city = city results << obj } - }catch(Exception e){ - println m.url+" 解析有问题!" - } +// }catch(Exception e){ +// println m.url+" 解析有问题!" +// } } return results @@ -410,7 +437,7 @@ class PaimaiTest { } } -// result.content = DataUtil.zip(s) + result.content = DataUtil.zip(doc.toString()) def el = JQueryUtils.select([".d-block table:eq(1)"], doc) if(!el) { @@ -549,7 +576,7 @@ class PaimaiTest { ret.each{k,v-> obj.put(k,v) } - + obj.source = "公拍网" if(dbo){ obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy index 244cc0d60..a7c7dccb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy @@ -1,12 +1,4 @@ -package com.fangjia.script.gj.paimai - -import com.fangjia.core.util.DataUtil -import com.fangjia.script.BaseBatch -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher -import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.gj.OpenApiUtils -import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.gj.paimai.PaimaiDataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -14,66 +6,266 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import org.jsoup.Jsoup import org.openqa.selenium.By -import org.openqa.selenium.JavascriptExecutor import org.openqa.selenium.WebDriver -import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.WebDriverWait import org.openqa.selenium.support.ui.ExpectedCondition import org.openqa.selenium.support.ui.ExpectedConditions -import org.openqa.selenium.support.ui.WebDriverWait +import com.fangjia.script.gj.OpenApiUtils -import java.util.concurrent.TimeUnit import java.util.regex.Matcher import java.util.regex.Pattern - +import com.fangjia.core.util.DataUtil +import com.fangjia.script.crawl.JQueryUtils System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); + + def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1" WebDriver driver = new ChromeDriver() -driver.get(url); -driver.findElementById("1").click() +run(url, driver) +//driver.get(url); +//driver.findElementById("1").click() +//driver.findElementById("search_sub").click() +// +//def provinces = getAllProvinceValue(driver.getPageSource()) +//for(def m : provinces){ +// def city_setting = driver.findElementById("city") +// //${m.province_id} +// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() +// +////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS); +////WebDriverWait wait = new WebDriverWait(driver,20) +////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1")))) +// Thread.sleep(10000) +// def doc = Jsoup.parse(driver.getPageSource()); +// def citys = getAllCity(doc) +// getDetailByCity(m.province, citys, driver) +//} -def city_setting = driver.findElementById("city") -city_setting.findElement(By.xpath("//option[@value='31']")).click() -driver.findElementById("search_sub").click() -//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS); -//WebDriverWait wait = new WebDriverWait(driver,20) -//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1")))) -Thread.sleep(10000) -def doc = Jsoup.parse(driver.getPageSource()); -def links = getLinks(doc) +def run(url, driver){ + def values = getAllCityRegionValue(url, driver) + def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"] + for(def m : values){ + if(!finishCity.contains(m.city)){ + driver.get(url); + driver.findElementById("1").click() + Thread.sleep(3000) + + def pro_setting = driver.findElementById("city") + pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() + Thread.sleep(3000) + + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(4000) + + def region_setting = driver.findElementById("city2") + region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click() + + driver.findElementById("search_sub").click() + Thread.sleep(10000) + getDetailByPage(m.province, m.city, m.region, driver) + } -def eles = driver.findElementsByClassName("pagecur1") -for(def ele : eles){ - ele.click(); - Thread.sleep(10000) - doc = Jsoup.parse(driver.getPageSource()); - links.addAll(getLinks(doc)) -} -def rets = [] -for(def durl : links){ - driver.get(durl); - doc = Jsoup.parse(driver.getPageSource()); - def res = parse(durl, doc); - if(res){ - rets << res } } -saveToDB(rets); + +def getAllCityRegionValue(def url, def driver){ + driver.get(url); + driver.findElementById("1").click() + driver.findElementById("search_sub").click() + + def provinces = getAllProvinceValue(driver.getPageSource()) + def values = [] + for(def m : provinces){ + def city_setting = driver.findElementById("city") + + city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() + + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def citys = getAllCity(doc) + def cityRegions = getAllCityByRegion(m, citys, driver) + values.addAll(cityRegions) + //getDetailByCity(m.province, citys, driver) + } + return values +} + +def getDetailByCity(province, citys, driver){ + for(def m : citys){ + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = getAllRegion(doc) + getDetailByRegion(province, m.city, regions, driver) + } +} + +def getAllCityByRegion(provinceMap, citys, driver){ + def datas = [] + for(def m : citys){ + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = getAllRegion(provinceMap, m, doc) + datas.addAll(regions) + } + return datas +} + +def getDetailByRegion(province, city, regions, driver){ + def links = [] + for(def r : regions){ + def city_setting = driver.findElementById("city2") + city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click() + driver.findElementById("search_sub").click() + Thread.sleep(10000) + + def stop = false; + while(!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + links.addAll(getLinks(r.region, doc)) + def eles = JQueryUtils.select("a:contains(下一页)", doc) + if(eles){ + driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() + Thread.sleep(3000) + }else{ + stop = true; + } + } + + } + def datas = parseDatas(province, city, links, driver) + saveToDB(datas); + println r.region +" "+ links.size() +} + +def getDetailByPage(province, city, region, driver){ + def links = [] + def stop = false; + while(!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + links.addAll(getLinks(doc)) + def eles = JQueryUtils.select("a:contains(下一页)", doc) + if(eles){ + driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() + Thread.sleep(3000) + }else{ + stop = true; + } + } + + def datas = parseDatas(province, city, region, links, driver) + println province +" "+city+" "+region +" "+ links.size() + saveToDB(datas); +} +def getAllProvinceValue(doc){ + def optionsEls = JQueryUtils.select("#city option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!="" && val!="0"){ + def province = eles.text() + rets << ["province":province, "province_id":val] -def parse(url, doc){ + } + + } + return rets +} + +def getAllCity(doc){ + def optionsEls = JQueryUtils.select("#city1 option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!=""){ + def province = eles.text() + rets << ["city":province, "city_id":val] + } + + } + return rets +} + +def getAllRegion(provinceMap, cityMap,doc){ + def optionsEls = JQueryUtils.select("#city2 option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!=""){ + def region = eles.text() + def regionMap = [:] + regionMap.province = provinceMap.province + regionMap.province_id = provinceMap.province_id + regionMap.city = cityMap.city + regionMap.city_id = cityMap.city_id + regionMap.region = region + regionMap.region_id = val + rets << regionMap + } + + } + return rets +} + +def parseDatas(province, city, region,links, driver){ + def datas = [] + for(def url: links){ + def doc = getDocument(url, driver) + def result = parse(city, region, url, doc) + if(result){ + result.province = province + result.city = city + result.region = region + datas << result + } + } + return datas +} + +def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc +} + +def parse(city, region, url, doc){ def result=[:] if(doc) { + def title = JQueryUtils.selectText(["#Title"], doc) result.title = title; result.address = title if(result.address){ - def rMap = OpenApiUtils.matchDistrict("上海",result.address) + def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true) if(rMap) { result.matchCity = rMap.city result.matchRegion = rMap.region @@ -99,6 +291,7 @@ def parse(url, doc){ def enrollCount = JQueryUtils.selectText([".bmnumber"], doc) if(enrollCount) { + enrollCount = enrollCount.replace("人报名","") if(enrollCount.isInteger()){ result.enrollCount = enrollCount.toInteger() } @@ -107,6 +300,7 @@ def parse(url, doc){ def callCount = JQueryUtils.selectText([".sc"], doc) if(callCount) { + callCount =callCount.replace("人收藏","") if(callCount.isInteger()){ result.callCount = callCount.toInteger() } @@ -114,6 +308,7 @@ def parse(url, doc){ def lookCount = JQueryUtils.selectText([".hits"], doc) if(lookCount) { + lookCount = lookCount.replace("次围观","") if(lookCount.isInteger()){ result.lookCount = lookCount.toInteger() } @@ -123,8 +318,8 @@ def parse(url, doc){ def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc) if(curPriceStr && curPriceStr.contains("起拍价")) { curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","") - if(curPriceStr.contains("万元")){ - curPriceStr = curPriceStr.replace("万元","") + if(curPriceStr.contains("万")){ + curPriceStr = curPriceStr.replace("万元","").replace("万","") if(curPriceStr.isInteger()){ result.startPrice = curPriceStr.toInteger()*10000 }else{ @@ -144,6 +339,44 @@ def parse(url, doc){ } } + def tableEles = JQueryUtils.select("#bg1", doc) + if(tableEles){ + def evaluatePriceStr = tableEles.select("td:contains(值:)").text() + if(evaluatePriceStr && evaluatePriceStr.contains(":")){ + def arrs = evaluatePriceStr.split(":") + if(arrs.size()==2){ + evaluatePriceStr = arrs[1] + } + } + + def depositPriceStr = tableEles.select("td:contains(金:)").text() + if(depositPriceStr && depositPriceStr.contains(":")){ + def arrs = depositPriceStr.split(":") + if(arrs.size()==2){ + depositPriceStr = arrs[1] + } + } + if(depositPriceStr){ + depositPriceStr = depositPriceStr.replace("万元","") + if(depositPriceStr.toString().isNumber()){ + result.depositPrice = depositPriceStr.toDouble()*10000 + } + } + + def courtStr = tableEles.select("td:contains(院:)").text() + if(courtStr && courtStr.contains(":")){ + result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size()) + } + def personStr = tableEles.select("td:contains(人:)").text() + if(personStr && personStr.contains(":")){ + result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size()) + } + + def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","") + if(countStr && countStr.contains("第") && countStr.contains("次")){ + result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次")) + } + } def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc) if(lis && lis.size()) { @@ -151,18 +384,23 @@ def parse(url, doc){ def str = li.select("span").text() if(str.contains("评估值")) { def v = li.select("span:eq(1)").text().replace("万元", "") - if(v.isInteger()) { - result.evaluationPrice = v.toInteger()*10000 - }else{ - result.evaluationPrice = (int)(v.toDouble()*10000) + if(v.isNumber()){ + if(v.isInteger()) { + result.evaluationPrice = v.toInteger()*10000 + }else{ + result.evaluationPrice = (int)(v.toDouble()*10000) + } } + }else if(str.contains("保证金")) { def v = li.select("span:eq(1)").text().replace("万元", "") - if(v) { - if(v.isInteger()) { - result.depositPrice = v.toInteger()*10000 - }else { - result.depositPrice = (int)(v.toDouble()*10000) + if(v.isNumber()) { + if (v) { + if (v.isInteger()) { + result.depositPrice = v.toInteger() * 10000 + } else { + result.depositPrice = (int) (v.toDouble() * 10000) + } } } }else if (str.contains("公告日期:")){ @@ -177,13 +415,32 @@ def parse(url, doc){ def trs = tables[1].select("tr") if(trs) { def detailMap = [:] + for(def tr : trs) { def d_tds = tr.select("td") + if(tr.text().contains("调查情况表")){ + continue + } + if(d_tds.size()>1){ + def lastEleStr = d_tds[d_tds.size()-1].toString() + if(lastEleStr.contains('height="42"')){ + d_tds.remove(d_tds[d_tds.size()-1]) + } + } + + if(d_tds.size()>3){ + continue + } if(d_tds && d_tds.size()>=2) { if(d_tds.size()==3) { - detailMap.put(d_tds[1].text(), d_tds[2].text()) + if(d_tds[1].text() && d_tds[2].text()){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + }else { - detailMap.put(d_tds[0].text(), d_tds[1].text()) + if(d_tds[0].text() && d_tds[1].text()){ + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } } }else { if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) { @@ -202,8 +459,6 @@ def parse(url, doc){ result.introduce = tables[1].toString() } - - def imgs =JQueryUtils.select("#bdjs11 img", doc) if(imgs) { def imgLinks = [] @@ -232,6 +487,20 @@ def parse(url, doc){ result.tradeAvgPrice = (int)(result.tradePrice/result.area) } + def s1 = JQueryUtils.selectText(["#pmgg"], doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + result.content = DataUtil.zip(doc.toString()) result.url = url @@ -246,7 +515,7 @@ def getLinks(doc){ if(eles){ for(def el :eles){ def href = el.attr("href") - urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length()) + urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length()) } } } @@ -264,13 +533,19 @@ def saveToDB(rets){ ret.each{k,v-> obj.put(k,v) } - - if(dbo){ - obj.remove("url") - paimaiResultColl.update(q, new BasicDBObject('$set', obj)) - }else{ - paimaiResultColl.save(obj) - } + obj.source= "人民法院诉讼资产网" + try{ + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + paimaiResultColl.save(obj) + } + }catch(Exception e){ + println "error:"+obj.url + } } } @@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) { for (def val : list) { def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); - if (maxArea) { - if (v.toDouble() < maxArea) { - area = v; + if(v && v.isNumber()){ + if (maxArea) { + if (v.toDouble() < maxArea) { + area = v; + break; + } + } else { + area = v break; } - } else { - area = v - break; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy new file mode 100644 index 000000000..1ce1c48c7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -0,0 +1,173 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CqfdcMatch { + + static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + static main(args) { + def task = new CqfdcMatch(); +// task.run() +// task.stat() + task.export2() + } + def export(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', false)) + + def field = new BasicDBObject() + field.put("city",1) + field.put("region",1) + field.put("district_name",1) + field.put("address",1) + field.put("type",1) + def dbcursor = loudongConn.find(q, field).batchSize(5000) + def keys = [] + def rets = [] + rets << ["城市","区域","名称","地址","类型"] + for(def d : dbcursor){ + def key = d.city +"##"+d.district_name + if(!keys.contains(key)){ + keys << key + rets << [d.city, d.region, d.district_name, d.address, d.type] + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据") + } + + def run(){ + def datas= findAddress2() + def results = match(datas); + update(results) + } + + + def export2(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', false)) + + def field = new BasicDBObject() + field.put("city",1) + field.put("name",1) + field.put("use_type",1) + def dbcursor = loudongFdcConn.find(q, field).batchSize(5000) + def keys = [] + def rets = [] + rets << ["城市","地址","类型"] + for(def d : dbcursor){ + def key = d.city +"##"+d.name + if(!keys.contains(key)){ + keys << key + rets << [d.city, d.name,d.use_type] + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据") + } + + def update(results){ + for(def m : results){ + def q = new BasicDBObject() + q.put("city", m.old_city) + q.put("district_name", m.old_name) + q.put("region",m.old_region) + + def udata = new BasicDBObject() + udata.put("match_city", m.match_city) + udata.put("match_region", m.match_region) + udata.put("match_name", m.match_name) + loudongConn.update(q, new BasicDBObject('$set',udata),false, true) + } + } + + def match(def datas){ + def rets = [] + for(def d : datas){ + def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + def map = [:] + map.match_city = jsonObject.get("city") + map.match_region = jsonObject.get("district") + map.match_name = jsonObject.get("name") + map.old_name = d.name + map.old_city = d.city + map.old_region = d.region + rets << map + } + } + return rets; + } + + def findAddress(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q) + def rets = [] + def keys = [] + for(def d : dbcursor){ + if(!keys.contains(d.name)){ + + keys << d.name + def map = [:] + map.city = d.city + map.name = d.name + rets << map + } + } + return rets; + } + + def findAddress2(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000) + def rets = [] + def keys = [] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.district_name + if(!keys.contains(key)){ + keys << key + def map = [:] + map.city = d.city + map.region = d.region + map.name = d.district_name + rets << map + } + } + return rets; + } + + + def stat(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', true)) + + def field = new BasicDBObject() + field.put("match_city",1) + field.put("match_region",1) + field.put("match_name",1) + def dbcursor = loudongConn.find(q, field).batchSize(2000) + def keys = [] + for(def d : dbcursor){ + def key = d.match_city+"##"+d.match_region+"##"+d.match_name + if(!keys.contains(key)){ + keys << key + } + } + + def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000) + for(def d : dbcursor2){ + def key = d.match_city+"##"+d.match_region+"##"+d.match_name + if(!keys.contains(key)){ + keys << key + } + } + println "小区:"+keys.size() + + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 97dc2e689..97af5b6dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "搜房"// - def citys = findCrawlCity(source, 202112); + def source = "58同城"// + def citys = findCrawlCity(source, 202202); for(def city : citys){ - def count1 = findCount(source, city, 202112); - def count2 = findCount(source, city, 202202); + def count1 = findCount(source, city, 202202); + def count2 = findCount(source, city, 202203); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 9362389ca..ee0edcfe2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -40,19 +40,17 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() - def cityList = [ + def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", + "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", + "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", + "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", + "淮北":"huaibei|9357","滁州":"chuzhou|10266", "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", - "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", + "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", - "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", + "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", "呼和浩特":"hu|811", - "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", - "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", - "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", - "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", - "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", - "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", - "淮北":"huaibei|9357","滁州":"chuzhou|10266", "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", @@ -76,7 +74,7 @@ class Esf58Crawl extends BaseBatch{ "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", - "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", + "儋州":"danzhou|10394", "新乡":"xx|1016","许昌":"xc|977", "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", @@ -98,7 +96,7 @@ class Esf58Crawl extends BaseBatch{ "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "益阳":"yiyang|10198","衡阳":"hy|914", "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", @@ -181,7 +179,10 @@ class Esf58Crawl extends BaseBatch{ "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", - "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + "威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", + "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"aihkou|2053", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|27 6","商丘":"sq|1029","大同":"dt|6964", ] static main(args) { @@ -191,37 +192,37 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); -// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] -// for(def city : citys){ -// def str = cityList.get(city) + def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"] + for(def city : citys){ + def str = cityList.get(city) + for(def i=0;i<1;i++){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } + } + } + +// cityList.each{city, str-> // for(def i=0;i<1;i++){ -// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); -// if(districts){ -// def arrs = str.split("\\|") -// def url = "https://"+arrs[0]+".58.com/xiaoqu/" -// distMap.put(city, districts) -// addCrawlKeys(city); -// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// if(citys.contains(city)){ +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// } // } // } // } - cityList.each{city, str-> - for(def i=0;i<1;i++){ -// if(citys.contains(city)){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } -// } - } - } - } @@ -230,6 +231,10 @@ class Esf58Crawl extends BaseBatch{ for(def i=0;i5){ + districtName = districtName.substring(0,3) + } for(def j=1;j<=districtName.length()-1;j++) { // if(j<=3){ def keyword = districtName.toString().substring(0, j); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 0ca7c1afb..279f5f3c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,10 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", - "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] +// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", +// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", -// def sourceMap = ["fang.com":"搜房"] + def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() From d7e8008e63a6f5ed5584fa8df8dc7c27fa670c14 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 18 Apr 2022 17:00:19 +0800 Subject: [PATCH 09/83] =?UTF-8?q?region=20block=E6=B7=BB=E5=8A=A0=E7=BB=8F?= =?UTF-8?q?=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/dingding/DingTalkUtils.groovy | 3 ++- .../poi/GetLngLatFromRegionBlock.groovy | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 795b1556a..e308339de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ; + // 要导出的月份第一天 + Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy new file mode 100644 index 000000000..c23db19ed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy @@ -0,0 +1,27 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +final DBCollection ZS_REGION_CONN = MongoUtils.getCollectionByName2("fangjia", "region_block", ZS_MONGODB_URL) + +ZS_REGION_CONN.find(new BasicDBObject()).batchSize(1000).each {obj -> + def city = obj.city + def org_name = obj.org_name + + List result = GaoDeUtils.searchPoi(city, org_name, org_name, null) + if (CollectionUtils.isNotEmpty(result)) { + def location = result.get(0).location + def lon = location.split(",")[0] + def lat = location.split(",")[1] + ZS_REGION_CONN.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set',new BasicDBObject("location", new BasicDBObject("lon", lon).append("lat", lat)))) + } +} \ No newline at end of file From 9a4e807e3cd0168cd0546a96bdc474ba68b3a101 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Tue, 19 Apr 2022 13:46:20 +0800 Subject: [PATCH 10/83] =?UTF-8?q?=E6=94=BE=E5=BC=80=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=B4=A6=E5=8F=B7=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index f6e9855f0..eb9325a57 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -665,18 +665,6 @@ public class PropertyController { return mav; } String userId = FangjiaApiUtil.parseUserId(model.getToken()); - - /**超级用户和上海交大账户不控制权限*/ - if (!"1007497".equals(userId) && !"1007632".equals(userId) && !"1007548".equals(userId) - && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) - && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) - && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) && !"1007757".equals(userId) - && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 - mav.setModel(BasicModel.buildUnAuthorizedModel()); - return mav; - } - District district = null; if (StringUtils.isNotBlank(model.getId())){ if(ObjectId.isValid(model.getId())){ From 3f9d3456d72b92eeb1abbc6e3cb0f8033021c37d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 19 Apr 2022 19:32:55 +0800 Subject: [PATCH 11/83] =?UTF-8?q?=E6=96=B0=E5=A2=9Eapi=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/TradeStatLog.groovy | 14 +-- .../gj/apiLog/ApiTimeOutAnalyBatch.groovy | 61 ++++++++++ .../gj/apiLog/ApiUserCountAnalyBatch.groovy | 111 ++++++++++++++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 6 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 6 files changed, 185 insertions(+), 13 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index afbaed5ee..18a865ad1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2021-12-01" - def date = "2022-01-01" - def endDate = "2022-02-01" - def newDate = "2022-03-01" + def startDate = "2022-01-01" + def date = "2022-02-01" + def endDate = "2022-03-01" + def newDate = "2022-04-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy new file mode 100644 index 000000000..b002b2c3b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.gj.apiLog + +import com.alibaba.excel.util.DateUtils +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.Calendar; + +@BatchOwner(Watcher.GaoJuan) +class ApiTimeOutAnalyBatch extends BaseBatch{ + + DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) + + static main(args) { + runBatch(new ApiTimeOutAnalyBatch(), args); + } + + @Override + public void process(Object args) { + def uriCountMap = findTimeOutUri() + sendMessage(uriCountMap) + } + + def sendMessage(uriCountMap){ + def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:" + def total = message.length() + uriCountMap.each{uri,count-> + if(count>=5){ + message = message + uri +"\n" + } + } + if(message.length()>total){ + println message + def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea" + DingDingMessageUtil.sendTextMessage(access_token, message) + } + } + + def findTimeOutUri(){ + def q = new BasicDBObject() + q.put("time", new BasicDBObject('$gte', 2000)) + def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5) + q.put("c_date", new BasicDBObject('$gte', startTime)) + def dbcursor = LOG.find(q) + def uriCountMap = [:] + for(def d : dbcursor){ + def count = uriCountMap.get(d.request_uri) + if(!count){ + count = 0 + } + uriCountMap.put(d.request_uri, count + 1) + } + return uriCountMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy new file mode 100644 index 000000000..16063d88b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.apiLog + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +@BatchOwner(Watcher.GaoJuan) +class ApiUserCountAnalyBatch extends BaseBatch{ + + static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver"); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl) + DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl) + + + static main(args) { + runBatch(new ApiUserCountAnalyBatch(), args); + } + + @Override + public void process(Object args) { + def rets = findAllUsers() + sendMessage(rets) + } + + + def sendMessage(def rets){ + def message = "【Api接口使用提醒】\n" + def total = message.length() + for(def r : rets){ + def type = "正式帐户" + if(r.type!=0){ + type = "测试帐户" + } + message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n" + } + if(message.length()>total){ + println message + def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea" + DingDingMessageUtil.sendTextMessage(access_token, message) + } + + } + + def findAllUsers(){ + def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'" + def rows = dbFangjia.rows(sql) + def rets = [] + for(def row : rows){ + def obj = findResources(row.id) + if(obj){ + def map = obj.total_count_map + def uris = [] + map.each{k,v-> + //def visit_count = findVisitCount(row.id, k) + def total_visit_count = findAllVisitCount(row.id, k) + println row.id +" "+k+" "+v+" "+total_visit_count + if(v && total_visit_count){ + def precent = v*0.8; + if(total_visit_count>=precent){ + uris << k + } + } + } + if(uris){ + rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test] + } + } + } + + return rets; + } + + def findAllVisitCount(userId, uri){ + def q = new BasicDBObject() + q.put("user_id", userId.toString()) + q.put("uri", uri) + q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date()))) + def dbcursor = VISIT_COLL.find(q) + def total = 0 + for(def obj : dbcursor){ + total = total +obj.visit_count + } + return total + } + + + def findVisitCount(userId, uri){ + def q = new BasicDBObject() + q.put("user_id", userId.toString()) + q.put("uri", uri) + q.put("visit_date", DateUtils.truncateDay(new Date())) + def obj = VISIT_COLL.findOne(q) + if(obj){ + return obj.visit_count + } + return null + } + + def findResources(def userId){ + def q = new BasicDBObject() + q.put("user_id", userId) + return AUTHORITY_COLL.findOne(q) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index d54ad9fb3..62fe7b599 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -182,7 +182,7 @@ public class BeiKeCrawlByChrome { } def areaStr = arrs[2].replace("平米", "") - if (areaStr) { + if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ + if(!e.text().contains("重庆")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index f1750cfa9..5c8d9c178 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -37,7 +37,7 @@ public class LianjiaCrawlByChrome { task.run() // task.run1() task.addDealRegion(); -// task.standardRegion() + task.standardRegion() } def run1(){ @@ -419,9 +419,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && e.text().contains("福州")) { +// if (!href.contains("fang.lianjia") && e.text().contains("福州")) { cityMap.put(e.text(), href) - } +// } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 279f5f3c1..12d8ebc60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"] + def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() From c86a0d63dfe1bf6443f7ee3bd5d45c9c62ca5098 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 19 Apr 2022 19:59:40 +0800 Subject: [PATCH 12/83] =?UTF-8?q?2000-=E3=80=8B3000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy index b002b2c3b..21dbfb7c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy @@ -44,7 +44,7 @@ class ApiTimeOutAnalyBatch extends BaseBatch{ def findTimeOutUri(){ def q = new BasicDBObject() - q.put("time", new BasicDBObject('$gte', 2000)) + q.put("time", new BasicDBObject('$gte', 3000)) def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5) q.put("c_date", new BasicDBObject('$gte', startTime)) def dbcursor = LOG.find(q) From f83b226d65f951636d4e6ae6acd1b961f97f44f1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Apr 2022 20:27:29 +0800 Subject: [PATCH 13/83] update --- .../ccbevaluation/BigdataDealBatch.groovy | 6 +-- .../ccbevaluation/ExportOracleData.groovy | 4 +- .../script/ccbevaluation/TradeStatLog.groovy | 2 +- .../fangjia/script/gj/YouDaDataParser.groovy | 8 ++-- .../gj/paimai/BeiKeCrawlByChrome.groovy | 44 ++++++++++++++----- .../gj/paimai/LianjiaCrawlByChrome.groovy | 9 +++- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 2 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 8 files changed, 51 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d34dedcc2..d1749e0ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; -// for(def city : citys){ -// addDealPriceMatch(city); -// } + for(def city : citys){ + addDealPriceMatch(city); + } // println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 9d06d7cef..060914122 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -38,8 +38,8 @@ class ExportOracleData { exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-02-19") - exportBigdataDeal("2022-02-01") + exportBigdataForsale("2022-03-19") + exportBigdataDeal("2022-03-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 18a865ad1..b0b02bde1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index b49996401..68494ea8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -42,17 +42,17 @@ class YouDaDataParser { static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search" - static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc" + static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc" static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-02-01" + static def statDate = "2022-03-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 62fe7b599..a6b739898 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome { System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def cityMap = getAllCityUrls(driver) + def citys = getAllCityUrls(driver) + citys = citys.reverse() + def start = 0 +// for(def i=0;i + for(def m : citys){ //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, city, cityUrl) - println cityUrl +" 详情链接:"+detailUrls.size() + def detailUrls = getPageDeail(driver, m.city, m.url) + println m.city +" 详情链接:"+detailUrls.size() // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome { break; } } + if(page>=100){ + stop = true; + break; + } page++; }else{ stop = true; @@ -337,17 +353,21 @@ public class BeiKeCrawlByChrome { driver.get("https://www.ke.com/city/") Thread.sleep(8000) def cityDoc = Jsoup.parse(driver.getPageSource()); - def cityEles = JQueryUtils.select(".city_province a", cityDoc) - def cityMap = [:] + def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) + def list = [] if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(!e.text().contains("重庆")){ - cityMap.put(e.text(), "https:"+href) + if(e.text().contains("重庆")){ + def cityMap = [:] + cityMap.city = e.text() + cityMap.url = "https:"+href + list << cityMap + //cityMap.put(e.text(), "https:"+href) } } } - return cityMap + return list } def getLogin(driver){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 5c8d9c178..6f99480ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -227,6 +227,10 @@ public class LianjiaCrawlByChrome { break; } } + if(page>=100){ + stop = true; + break; + } page++; }else{ stop = true; @@ -416,12 +420,13 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] + def citys= ["长春","吉林"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") -// if (!href.contains("fang.lianjia") && e.text().contains("福州")) { + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) -// } + } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ee0edcfe2..ace3084c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -192,7 +192,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"] + def citys = ["德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 12d8ebc60..fec88b8c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","5i5j.com":"我爱我家"] + def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() From 73dcc7f14c38d3dc195229cf46dc9143d880ccb1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Apr 2022 20:37:02 +0800 Subject: [PATCH 14/83] update --- .../fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 6 +++--- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d1749e0ce..d34dedcc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; - for(def city : citys){ - addDealPriceMatch(city); - } +// for(def city : citys){ +// addDealPriceMatch(city); +// } // println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 421a0e4f7..e262bd7c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202112 - def endDate = 202202 + def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202201 + def endDate = 202203 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From adadf832e33b465512e6268bb9d0ba1d553bd444 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Apr 2022 17:29:00 +0800 Subject: [PATCH 15/83] =?UTF-8?q?=E5=A2=9E=E5=8A=A02=E4=B8=AA=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/level/DistrictIndexBatch.groovy | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index e6046e703..dc090f531 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -58,6 +58,8 @@ class DistrictIndexBatch extends BaseBatch{ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM") + static main(args) { runBatch(new DistrictIndexBatch(), args); } @@ -71,7 +73,7 @@ class DistrictIndexBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def code = cityCodeDatas.get(city) def city_avg_total_price = null if(city!="上海"){ @@ -84,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } @@ -131,7 +133,7 @@ class DistrictIndexBatch extends BaseBatch{ map.total_price = total_price //map.comp_value = potentiality //map.avg_total_price = total_price_index - map.month = "2022-03" + map.month = month map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") //datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] @@ -229,7 +231,7 @@ class DistrictIndexBatch extends BaseBatch{ map.comp_value = potentiality map.avg_total_price = total_price_index - map.month = "2022-03" + map.month = month map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") map.finish_date = year @@ -240,6 +242,15 @@ class DistrictIndexBatch extends BaseBatch{ if(rMap.sell_avg_price && sqmetre_rent){ map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger() } + if(d.household_count){ + map.household_ount = d.household_count + } + if(d.lat){ + def lMap = [:] + lMap.lon = d.lng/10000000000.0 + lMap.lat = d.lat/10000000000.0 + map.location = lMap + } def newUrl = targetUrl +"_doc/"+map.id httpSaveData(newUrl, map) From 3d525b2e6cd2f6d1144c26db1eb177b44c3462c6 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sun, 24 Apr 2022 11:01:53 +0800 Subject: [PATCH 16/83] =?UTF-8?q?household=5Fcount=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index dc090f531..bd387ed77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -243,7 +243,7 @@ class DistrictIndexBatch extends BaseBatch{ map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger() } if(d.household_count){ - map.household_ount = d.household_count + map.household_count = d.household_count } if(d.lat){ def lMap = [:] From 94d15915b6a70e65d99a2423f162536eac07135b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 6 May 2022 15:28:55 +0800 Subject: [PATCH 17/83] update --- .../ccbevaluation/BigdataBuildingBatch.groovy | 84 ++-- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 18 +- .../script/ccbevaluation/TradeStatLog.groovy | 8 +- .../ccbevaluation/data/CcbDistrictToDB.groovy | 296 +++++++++++- .../chongqing/BlankEvaluationBatch.groovy | 8 +- .../script/chongqing/BlankMatch.groovy | 64 +-- .../chongqing/MatchEvaluateRegionTest.groovy | 239 ++++++++++ .../script/chongqing/MatchEvaluateTest.groovy | 280 +++++++++++ .../script/commData/CommonDatautils.groovy | 18 + .../script/common/DataHandleUtils.groovy | 6 +- .../script/gj/chongqingblank/DataMatch.groovy | 4 +- .../script/gj/crawl/AnjukeDetailCrawl.groovy | 246 ++++++++++ .../script/gj/crawl/CommonCrawl.groovy | 29 ++ .../script/gj/data/TempMatchDistrict.groovy | 2 +- .../gj/district/FangjiaClientUtils.groovy | 17 + .../script/gj/meituan/MatchTest.groovy | 354 ++++++++++++++ .../script/gj/paimai/Caa123Crawl.groovy | 28 +- .../fangjia/script/gj/paimai/DataUtil.groovy | 25 +- .../fangjia/script/gj/paimai/IcbcCrawl.groovy | 348 ++++++++++++++ .../script/gj/paimai/JdPaimaiCrawl.groovy | 9 +- .../script/gj/paimai/TaobaoPaimaiCrawl.groovy | 450 ++++++++++++++++-- .../fangjia/script/loudong/CqfdcMatch.groovy | 128 ++++- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 2 +- .../fangjia/script/util/CommonUtils.groovy | 42 ++ .../com/fangjia/script/util/DbUtils.groovy | 37 ++ .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 27 files changed, 2604 insertions(+), 144 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 42c580735..6ef05438e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{ def q = new BasicDBObject() q.put("city", city) q.put("status", 1) +// q.put("name","辽源一村") def dbcursor = LOU_DONG_STAT.find(q) def map = [:] + def set = new HashSet() for(def db : dbcursor){ + def list = map.get(db.region+"##"+db.name) if(!list){ list = [] map.put(db.region+"##"+db.name, list) } - list << db + if(db.house_num && db.house_num!="--"){ + def key = db.region+"##"+db.name +"##"+db.house_num + if(!set.contains(key)){ + set << key + list << db + } + } } return map } def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){ def rets = [] + def keys = [] distMap.each{k,dist-> def buildings = buildingMap.get(k) if(buildings){ @@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{ def ret = [] def areaData = distCodeMap.get(obj.region) if(areaData){ - ret << cityCodeDatas.get(obj.city) - ret << obj._id.toString() - ret << obj.house_num - ret << "" - ret << "" - ret << areaData.district_code - ret << dist._id.toString() - if(obj.build_year){ - ret << obj.build_year - }else{ + def key = dist._id.toString() +"##"+obj.house_num + if(!keys.contains(key)){ + keys << key + ret << cityCodeDatas.get(obj.city) + ret << obj._id.toString() + ret << obj.house_num + ret << obj.house_num ret << "" - } - def arrs = CcbCodeUtils.parseBuildType(dist.build_type) - if(arrs){ - ret << arrs[0].trim() - }else{ - ret << '' - } + ret << areaData.district_code + ret << dist._id.toString() + if(obj.build_year){ + ret << obj.build_year + }else{ + ret << "" + } + def arrs = CcbCodeUtils.parseBuildType(dist.build_type) + if(arrs){ + ret << arrs[0].trim() + }else{ + ret << '' + } + + ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) + if(obj.floors){ + ret << obj.floors + }else{ + ret << "" + } - ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) - if(obj.floors){ - ret << obj.floors - }else{ ret << "" - } - - ret << "" - ret << "" - ret << "" + ret << "" + ret << "" // if(obj.elevators && obj.elevators>=1){ // ret << "1" // }else{ // ret << "0" // } - ret << "" + ret << "" // if(obj.elevators && obj.elevators>=1) { // ret << obj.elevators//电梯数 // }else{ // ret << "" // } - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << "1" - rets << ret + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << "1" + rets << ret + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 8a8b4a7dc..98fbc2d09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -367,7 +367,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","特房锦绣一方") +// q.put("name","辽源一村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index f75d28656..d7840e5bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,8 +25,8 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); -// task.deleteThirdPartyData() - task.delDealByPe() + task.deleteThirdPartyData() +// task.delDealByPe() } def delDealByPe(){ @@ -370,26 +370,26 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = ["东莞"] //,"杭州""广州", + def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202110 + def month = 202203 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202111) + def rets = findPeGteThirdParty(city, 202204) def total = 0 for(def m : rets){ def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) } - println city +" "+total + println city +" "+rets.size() } } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50" def rets = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance236.eachRow(sql,{r-> def map = [:] map.commid = r.commid map.price = r.target_avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index b0b02bde1..3ed13ebfc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 -// task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -481,8 +481,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2022-02-01" - def endDate = "2022-03-01" + def startDate = "2022-03-01" + def endDate = "2022-04-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 5df3fb0fd..2db54e10a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.DateUtil import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.mongodb.BasicDBObject +import java.util.concurrent.CopyOnWriteArrayList + class CcbDistrictToDB { static main(args) { def task = new CcbDistrictToDB(); // task.run() - task.run1() +// task.run1() +// task.meituanRun() +// task.meituanSqrentRun() +// task.updateDistrict() + task.updateDongguangRegion() } + + def updateDongguangRegion(){ + def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd") + def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"]) + for(def d : dist){ + def t = d.c_date + if(t.getTime()>cDate.getTime()){ + + } + } + } + + def updateDistrict(){ + def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"] + def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd") + def index = 0 + for(def city : citys){ + def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"]) + for(def d : dist){ + def t = d.c_date + if(t.getTime()>cDate.getTime()){ + if(d.name.contains("(")){ + def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1) + def newName = d.name.replace(delstr,"") + index++ + println index+" "+d.city+" "+d.name+" "+newName + def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district") + if(!obj){ + println "update" + def udata = new BasicDBObject() + udata.put("name", newName) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + + d.name = newName + addPrice(d) + + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0) + }else{ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) + } + } + } + } + } + } + + def meituanSqrentRun(){ + ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def city = d['F'] + def region = d['G'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion) { + def name = d['H'] + name = nameFileter(name) + if(name){ + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0) + } + } + },0); + } + + def meituanRun(){ + def index = 0//美团需入库小区合并后.xlsx + def keys = new CopyOnWriteArrayList() + + ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def city = d['F'] + def region = d['G'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['H'] + name = nameFileter(name) + + def formatAddress = null + if(d['E']){ + formatAddress = AddressUtil.formatAddress(city, d['E']) + } + + if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ + def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) + def map = findStandardObject2(city,region, name, d['E'], rets); + if(map && map.address){ + formatAddress = AddressUtil.formatAddress(city, map.address) + if(!formatAddress){ + if(map.address.contains("交叉口")){ + formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) + }else if(map.address.contains("交叉")){ + formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) + } + } + } + } + + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + + + println index+" "+data + + def stop = false + while (!stop){ + if(data._id.toString()){ + addPrice(data) + stop = true; + } + } + + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + },0); + + + + } + + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + def run1(){ def index = 0 - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d -> + + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d -> def city = d['A'] def region = d['B'] def newRegion = RegionData.formatRegion(city, region) @@ -71,6 +289,7 @@ class CcbDistrictToDB { if(name){ def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") if(!dist){ + def data = new BasicDBObject(); data.put("name_cat",0) data.put("status",0) @@ -221,4 +440,77 @@ class CcbDistrictToDB { name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") return name; } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy index 88f5c3bdf..e45fc62c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy @@ -16,7 +16,9 @@ import com.mongodb.DBObject import groovy.sql.Sql import org.apache.lucene.queryparser.classic.QueryParser - +/** + * 重庆估值测试--2小时 + */ class EvaluationBatch { public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; @@ -27,8 +29,8 @@ class EvaluationBatch { static main(args) { def task = new EvaluationBatch() // task.run() - task.run1(); -// task.run2() +// task.run1(); + task.run2() } def run2(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy index 75c731a7f..77b853dad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy @@ -8,42 +8,17 @@ import com.fangjia.script.util.OpenApiClient class BlankMatch { static main(args) { def task = new BlankMatch() -// task.run() + task.run() // task.run1() // task.run2() - task.run3() - } - - def run3(){ - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> - if(!row['C']){ - def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") - if(dist){ - row['C'] = dist._id.toString() - } - } - },0) - } - def run2(){ - def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> - if(!row['C']){ - for(def d : datas){ - if(d['B'] == row['B']){ - row['C'] = d['C'] - row['D'] = d['D'] - row['E'] = d['E'] - row['F'] = d['J'] - break; - } - } - } - },0) +// task.run3() } def run() { //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); - ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + //def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx"; + def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> if(!row['C']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) if(jsonObject != null && jsonObject.precise!=-1) { @@ -54,8 +29,6 @@ class BlankMatch { row['G'] = jsonObject.get("address") } } - - }); } @@ -81,4 +54,31 @@ class BlankMatch { // } }); } + + def run3(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") + if(dist){ + row['C'] = dist._id.toString() + } + } + },0) + } + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + for(def d : datas){ + if(d['B'] == row['B']){ + row['C'] = d['C'] + row['D'] = d['D'] + row['E'] = d['E'] + row['F'] = d['J'] + break; + } + } + } + },0) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy new file mode 100644 index 000000000..bae011454 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -0,0 +1,239 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList + +class MatchEvaluateRegionTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new MatchEvaluateRegionTest() + task.run()//小区匹配 + task.run1()//地图匹配 +// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 + task.addPrice() + task.addAuthor() + } + def run(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['F'] = jsonObject.get("id") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("name") + row['J'] = jsonObject.get("address") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def run1(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['F'] = jsonObject.get("id") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("name") + row['J'] = jsonObject.get("address") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def addPrice(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['L']){ + def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) + if(priceObj){ + row['M'] = priceObj.avg_price + row['N']= (int)(priceObj.avg_price * row['E']) + } + + def avg_price = findAvgPrice(row['M']) + if(row['M'] && avg_price){ + def weekendPrice = row['M'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + + } + + def addAuthor(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def total = 747 + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(index < avg){ + row['P'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + }); + } + + + def addDistrict(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def flag = d['M'] + if(!flag){ + def city = d['E'] + def region = d['F'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['G'] + name = nameFileter(name) + + def formatAddress = null + def address = d['H'] + if(address){ + formatAddress = AddressUtil.formatAddress(city, address) + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + addPrice(data) + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + } + },0) + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name.contains("(")){ + def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1) + name = name.replace(delstr,"") + } + return name; + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy new file mode 100644 index 000000000..e0a41a599 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -0,0 +1,280 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList +import com.fangjia.script.gj.district.FangjiaClientUtils + +class MatchEvaluateTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx" + def total = 1500 + + static main(args) { + def task = new MatchEvaluateTest() +// task.run()//小区匹配 +// task.run1()//地图匹配 +//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 +// task.addPrice() +// task.addArroundPrice() +// task.addAuthor() + + task.stat() + } + + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A'] && !row['K']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['K'] = priceObj.unitPrice + row['L'] = (int) (priceObj.unitPrice * row['C']) + row['O'] = "周边价格" + } + } + }); + } + + + def run(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['D'] && row['B']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['D'] = jsonObject.get("id") + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("address") + row['I'] = jsonObject.get("propertyType") + row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def run1(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['D'] && row['B']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['D'] = jsonObject.get("id") + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("address") + row['I'] = jsonObject.get("propertyType") + row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def addPrice(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['J'] && row['E']){ + def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G']) + if(priceObj){ + row['K'] = priceObj.avg_price + row['L']= (int)(priceObj.avg_price * row['C']) + } + + + def avg_price = findAvgPrice(row['D']) + if(row['K'] && avg_price){ + def weekendPrice = row['K'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + + } + + def addAuthor(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['B']){ + if(index < avg){ + row['N'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + + def addDistrict(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def flag = d['M'] + if(!flag){ + def city = d['E'] + def region = d['F'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['G'] + name = nameFileter(name) + + def formatAddress = null + def address = d['H'] + if(address){ + formatAddress = AddressUtil.formatAddress(city, address) + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + addPrice(data) + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + } + },0) + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name.contains("(")){ + def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1) + name = name.replace(delstr,"") + } + return name; + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } + + def stat(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1) + ExcelUtils.multiThreadingModify(path, { d -> + if(d['B']){ + for(def row : datas){ + if(d['B']==row['B']){ + if(row['K']){ + d['P'] = row['K'] + break + } + } + } + } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 239c9f83c..d6ab63366 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -830,6 +830,7 @@ class CommonDatautils { keys.put("name_cat", 1); keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) + keys.put("sid", 1) DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ @@ -3166,5 +3167,22 @@ class CommonDatautils { } return avg_price; } + + static def findCityAvgRentRadio(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + + def dbcursor = SQMETRE_RENT_COLL.find(q) + def avg_price = null + def radios = [] + for(def d : dbcursor){ + if(d.price_to_rent){ + radios << d.price_to_rent.toInteger() + } + } + def avg_radios = (int)(radios.sum()/radios.size()) + return avg_radios; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy index 7d1ef8018..7fe3204dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy @@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils; class DataHandleUtils { - static main(args){ - - } + static def getStandardRegion(def regionMap, def region){ def newRegion = null; @@ -94,6 +92,8 @@ class DataHandleUtils { BigDecimal deSource = new BigDecimal(dSource); return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue(); } + + static def trimChar(String c) { try { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 9f84f6479..418b6c102 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -25,11 +25,11 @@ class DataMatch { static main(args) { def task = new DataMatch() // task.run1() - task.run2() +// task.run2() // task.run3() // println task.findAddress("昌州街道棠香路") -// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy new file mode 100644 index 000000000..d4f3001b3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy @@ -0,0 +1,246 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeDetailCrawl { + DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") + static main(args) { + def task = new AnjukeDetailCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://chongqing.anjuke.com/community/?from=navigation" + WebDriver driver = new ChromeDriver() + def regions = getRegions(driver, url) + getAllRegions(driver, regions) + + } + + + def getAllRegions(driver, regions){ + for(def url : regions){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def total = getTotal(doc) + if(total) { + if (total > 1250) { + + } else { + def links = getAllLinks(url, driver) + parse(links,driver) + } + } + } + } + + def getAllLinks(url, driver){ + def stop = false + def urls = [] + while(!stop) { + def doc = Jsoup.parse(driver.getPageSource()); + def details = JQueryUtils.select(".list-cell a.li-row", doc) + + for (def d : details) { + urls << d.attr("href") + } + def nextEle = JQueryUtils.select(".next", doc) + if (nextEle) { + driver.findElement(By.cssSelector(".next")).click() + }else{ + stop = true + } + if(urls.size()>20){ + stop = true + } + } + + return urls + } + + def getTotal(doc){ + + def totalText = JQueryUtils.selectText(".total-info", doc) + if(totalText){ + def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim() + if(total && total.isNumber()){ + return total.toInteger() + } + } + return null; + } + + def getRegions(driver, url){ + driver.get(url) + Thread.sleep(3000) + def doc = Jsoup.parse(driver.getPageSource()); +// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) +// if(loginIfm){ +// def href = loginIfm[0].attr("src") +// driver.get(href) +// driver.findElementById("pwdTab").click() +// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框 +// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框 +// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。 +// elemUsername.sendKeys("17740818975"); +// elemPassword.sendKeys("12qwaszx"); +// btn.submit(); // 点击行为; +// +// def regionEles = JQueryUtils.select(".region-parents li a", doc) +// def urls = [] +// for(def e : regionEles){ +// urls << e.attr("href") +// } +// return urls; +// } + def regionEles = JQueryUtils.select(".region-parents li a", doc) + def urls = [] + for(def e : regionEles){ + urls << e.attr("href") + } + return urls; + } + + def parse(def links, driver){ + def rets = [] + for(def link : links){ + def district = parsing(link, driver) + if(district){ + rets << district + + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + rets = [] + } + } + } + if (rets.size()> 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + } + } + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = detailColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + detailColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + detailColl.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + return add + } + + + + def parsing(url, driver){ + + def district = [:] + def stop = false + def count = 0 + while(!stop){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def headers = JQueryUtils.select(".crumbs a", doc) + if(headers){ + if (headers && headers.size() >= 3) { + district.city = headers[1].text().replace("小区", "") + district.region = headers[2].text().replace("小区", "") + district.name = headers[headers.size() - 1].text().replace("二手房", "") + } + + + district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc) + + def priceStr = JQueryUtils.selectText(".average", doc) + + if (priceStr && priceStr.isNumber()) { + district.avg_price = priceStr.toInteger() + } + + def infoDiv = JQueryUtils.select(".info-list", doc) + district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) + district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) + district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) + district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) + + def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() + if (houseCountStr) { + houseCountStr = houseCountStr.replace("户", "") + if (houseCountStr) { + district.house_count = houseCountStr.toInteger() + } + } + + district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) + district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) + def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) + if(carSiteStr){ + if(carSiteStr.contains("(")){ + carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("(")) + } + if(carSiteStr && carSiteStr.isNumber()){ + district.car_site = carSiteStr + } + } + district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) + district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) + district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) + district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) + district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.url = url + district.c_date = new Date() + district.m_date = new Date() + district.root = "anjuke.com" + + stop = true; + }else{ + Thread.sleep(3000) + count++ + } + if(count>10){ + stop = true; + } + } + + return district + } + + String getValue(String s){ + if(s && s.contains("暂无")){ + return null + } + return s + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy new file mode 100644 index 000000000..f19c4cd3c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.gj.crawl + +import org.jsoup.Jsoup + +class CommonCrawl { + static def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy index a36cf12a7..13b8fdde9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -17,7 +17,7 @@ class TempMatchDistrict { } def stat(){ - def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"] + def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] def total = 0 for(def city : citys){ def count = CommonDatautils.findAllDistrictCount("district", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 44f6d4a9b..12d2f77b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -83,6 +83,23 @@ class FangjiaClientUtils { return jsonObj; } + public static def arroundPrice(city,address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100" +// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static Map JsonToMap(JSONObject json) { Map columnValMap = new HashMap(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy new file mode 100644 index 000000000..6b4872d19 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy @@ -0,0 +1,354 @@ +package com.fangjia.script.gj.meituan + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId + +class MatchTest { + static main(args) { + def task = new MatchTest() +// task.run() +// task.addNum() + task.dongguang() +// task.meituanDel() + } + + def meituanDel(){ + def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" + def index = 0 + ExcelUtils.modify(path, { row -> + def flag = row['O'] + if(flag){ + def city = row['F'] + def region = row['G'] + def name = nameFileter(row['H']) + def obj = null + if(city == "东莞"){ + obj = CommonDatautils.findDistrict(city, null, name, "district") + }else{ + obj = CommonDatautils.findDistrict(city, region, name, "district") + } + if(obj){ + index++ + println index +" "+ city +' '+ region+" "+name + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(obj._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + },0); + } + + def dongguang(){ + def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx + ExcelUtils.modify(path, { row -> + def region = null + if(row['B'] && row['B']=="NULL" ){ + region = null + } + def name = nameFileter(row['H']) + def address = row['E'] + if (name && name.contains("湖畔新家")) { + + def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district") + if(obj){ + def json = GaoDeApiUtils.geo(row['F'], address) + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + def twinshop = getStreetInfo(m.location) + if(twinshop){ + def newRegion = RegionData.formatRegion("东莞",twinshop) + if(newRegion){ + def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district") + if(dbObj){ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id)) + }else{ + println "update" + def udata = new BasicDBObject() + udata.put("region", newRegion) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + } + // println address +" "+twinshop + } + } + } + } + },0) + + } + + def getStreetInfo(def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + } + } + } + return township; + } + + def addNum(){ + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx + def index = 0 + ExcelUtils.modify(path, { row -> + index++ + row['A'] = index + },0) + } + + def run() { + def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + def radioMap = [:] + for(def city : citys){ + +// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); +// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) +// radioMap.put(city, avg_radio) + + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + + } + + + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + def region = null + if(row['C'] && row['C']=="NULL" ){ + region = null + } + def name = row['D'] + def address = row['F'] + def city = RegionData.formatCity(row['B']) + + if(name && !row['N']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } + if(rMap.level){ + row['P'] = rMap.level + } + } + } + } + + if (name && row['N']) { +//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name) +// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null) +// if (jsonObject != null && jsonObject.precise != -1) { +// row['F'] = jsonObject.get("city") +// row['G'] = jsonObject.get("district") +// row['H'] = jsonObject.get("name") +// row['I'] = jsonObject.get("propertyType") +// row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") +// } + + if(row['G'] && row['H'] && row['I']){ +// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H']) +// if(priceObj){ +// row['M'] = priceObj.avg_price +// } +// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H']) +// if(sqmetreRentPrice){ +// row['N'] = sqmetreRentPrice +// } + + if(row['N'] && !row['O']){ + def avg_rent_radio = radioMap.get(city) + row['O'] = (int)(row['N']/avg_rent_radio) + } + } + + } + }) + } + + def getArroundDistrictPriceByAddress(def city, def address, def levels){ + def json = GaoDeApiUtils.geo(city, address) + def location = null + def detail = new BasicDBObject() + def level = null + def rank = null + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + location = m.location + level = m.level + } + } + if(location){ + def arrs = location.split(",") + detail.lng = arrs[0].toString().toDouble()*10000000000L + detail.lat = arrs[1].toString().toDouble()*10000000000L + } + if(detail.lng){ + detail.city = city + + List list = DbUtils.findAroundDistrictByRadius(detail, 3000) + if(list && list.size()>10){ + list = sortByDistance(detail, list); + list = list[0..9] + } + def arroundAvgPrice = getArroundAvgPrice(list) + if(arroundAvgPrice){ + + def mod = Integer.MAX_VALUE; + for(def d : levels){ + def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice) + if(newMod o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + return datas; + } + + def getArroundAvgPrice(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + if(prices){ + def avg_price = (prices.sum()/prices.size()).toInteger() + return avg_price; + } + return null + } + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + return name; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index afaae338f..fb358e503 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -42,7 +42,6 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() def proviceMap = getDocument(driver); def citys = getCity(driver,proviceMap) @@ -59,9 +58,11 @@ def getDetailLinks(driver, cityList){ citys.each{city,id-> def urls = [] if(!zxcitys.contains(cityMap.provice)){ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) }else{ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) } def doc = Jsoup.parse(driver.getPageSource()); @@ -89,7 +90,7 @@ def getDetailLinks(driver, cityList){ } } - def rets = batchParse(urls); + def rets = batchParse(urls, driver); saveToDB(rets); println city +" "+urls.size() } @@ -124,7 +125,7 @@ def getCity(driver, proviceMap){ for(def c : cityEles){ def cityid = c.attr("data-cityid") def city = c.text() - if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ + if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ cityMap.put(city, cityid) } @@ -149,7 +150,7 @@ def getCity(driver, proviceMap){ def getDocument(driver){ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport="); WebDriverWait wait = new WebDriverWait(driver,20) wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); def doc = Jsoup.parse(driver.getPageSource()); @@ -159,7 +160,7 @@ def getDocument(driver){ for(def ele : proviceEles){ def provice = ele.text() def value = ele.attr("data-proviceid") - if(value && provice!="北京市" && provice!="天津市"){ + if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){ provinceMap.put(provice, value) } } @@ -167,8 +168,8 @@ def getDocument(driver){ return provinceMap; } -def batchParse(def urls){ - WebDriver driver = new ChromeDriver() +def batchParse(def urls, driver){ + def ret = [] for(def map :urls){ def doc = getDocument(map.url, driver) @@ -280,7 +281,7 @@ def parse(province, city, url,doc){ result.lookCount = JQueryUtils.selectText(".look_count span", doc) def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc) - if(startDateStr){ + if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){ startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() } @@ -329,6 +330,13 @@ def parse(province, city, url,doc){ result.tradeAvgPrice = (int) (result.tradePrice / result.area) } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + result.content = DataUtil.zip(doc.toString()) result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() result.url = url diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy index 8cce171c6..7e81c3afd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -16,6 +16,27 @@ import java.util.regex.Matcher import java.util.regex.Pattern class DataUtil { + + + static def checkMinDate(curDate, checkDate){ + if(curDate){ + def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd") + if(curDate.getTime()maxDate.getTime()){ + return null + } + } + return curDate + } static def getAreaNumber(def input, def maxArea) { Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)') Matcher matcher = pattern.matcher(input) @@ -112,10 +133,10 @@ class DataUtil { if (list.size() >= 1) { String s1 = list.get(0) if (s1.contains("万元")) { - s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "") + s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","") return (int) (s1.toDouble() * 10000) } else { - s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "") + s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","") if (s1.isInteger()) { return s1.toInteger() } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy new file mode 100644 index 000000000..b432108a2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy @@ -0,0 +1,348 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import javafx.scene.input.DataFormat +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class IcbcCrawl { + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new IcbcCrawl(); + task.run(); + } + + def run(){ + def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + driver.get(url) + driver.findElementById("category_SM310000000000000006").click() + Thread.sleep(3000) + + + def urls = getAllDetailLinks(driver) + println "共:"+urls.size() + + detailCrawling(urls, driver) + + } + + def getAllDetailLinks(driver){ + def stop = false + def urls = [] + while (!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc) + if(eles){ + for(def e : eles){ + def href = e.select(".li_2_name a").attr("href") + urls << "https://gf.trade.icbc.com.cn"+href + } + def nextPage = JQueryUtils.select("a:contains(下一页)", doc) + if(nextPage){ + driver.findElement(By.linkText("下一页")).click(); + }else{ + stop = true + } + }else{ + stop = true + } +// if(urls.size()>=10){ +// stop = true +// } + } + return urls; + } + + def detailCrawling(def urls, driver){ + def rets = [] + for(def url : urls){ + driver.get(url) + Thread.sleep(5000) + def doc = Jsoup.parse(driver.getPageSource()); + def result = parser(doc, url) + if (result) { + rets << result + println rets.size() + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + } + + def parser(doc, url){ + def result = [:] + if(doc){ + def title = JQueryUtils.selectText("#prodnameb", doc) + def id = url.substring(url.indexOf("="), url.length()) + if(title){ + + def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc) + if(cityStr && cityStr.contains("...")){ + def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc) + if(cityEles){ + cityStr = cityEles[0].attr("title") + } + } + if(cityStr && cityStr.contains("省")){ + result.province = cityStr.substring(0, cityStr.indexOf("省")) + result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length()) + }else{ + if(cityStr.contains("自治区")){ + result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3) + result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length()) + }else{ + result.city = cityStr.substring(0, cityStr.indexOf("市")) + } + } + + if(!result.city){ + return null + } + result.title = title + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + + result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","") + + def t1 = JQueryUtils.selectText("#pricep", doc) + if(t1 && t1.contains("原因")){ + result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length()) + } + result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc) + def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#visitorCount"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#remindPPNum"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim() + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc) + if(startPriceEles) { + def startPriceStr = startPriceEles[0].attr("title") + if(startPriceStr){ + startPriceStr = startPriceStr.replace(",", "").replace("¥","") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + } + + } + + + def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(addPriceEles){ + def addPriceStr = addPriceEles[0].attr("title") + if(addPriceStr){ + addPriceStr = addPriceStr.replace(",", "").replace("¥","") + if(addPriceStr) { + result.addPrice = addPriceStr.toDouble().toInteger() + } + } + } + + def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc) + if(evaluationPriceEles){ + def evaluationPriceStr = evaluationPriceEles[0].attr("title") + if(evaluationPriceStr){ + evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","") + if(evaluationPriceStr.isInteger()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(evaluationPriceStr.toDouble()) + } + } + } + + def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(depositPriceEles){ + def v = depositPriceEles[0].attr("title") + if(v){ + v = v.replace(",", "").replace("¥","") + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + } + } + + result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc) + result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc) + result.court = JQueryUtils.selectText("#goToCourt", doc) + def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc) + if(mobileEles){ + result.mobile = mobileEles[0].attr("title") + } + + def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + + def trs = JQueryUtils.select([".rygtable table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + if(d_tds[2].text() && d_tds[2].text()!="/"){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + + }else { + if(d_tds[1].text() && d_tds[1].text()!="/") { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs =JQueryUtils.select("#scroll li img", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("src") + if(href.contains("icbc.com")) { + imgLinks << "https:"+href + } + } + result.imgUrls = imgLinks + } + + + def detailStr = JQueryUtils.selectText(".detail_list", doc) + if(detailStr){ + def area = DataUtil.getAreaNumber(detailStr, 5000) + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = detailStr.toString() + result.url = url + result.mDate = new Date() + result.source = "融e够" + return result; + } + + } + return null + } + + def getCallCount(id){ + def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id + def s = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(s){ + JSONObject jsonObject = JSONObject.fromObject(s); + println jsonObject + } + } + + + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy index aac9c7315..3ea9f1d4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -39,8 +39,6 @@ class JdPaimaiCrawl { println "共:"+ids.size() parserList( existsIds,ids, driver) - - } def getList(driver){ @@ -478,6 +476,13 @@ class JdPaimaiCrawl { } } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString() result.url = url diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy index f0b328f11..68b73d58e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy @@ -1,7 +1,12 @@ package com.fangjia.script.gj.paimai import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.DateUtils +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver class TaobaoPaimaiCrawl { static main(args) { @@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl { task.run(); } def run(){ - def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1" - def regionUrls = regionCrawl(seedUrl); - def pageUrls = pageUrlCrawl(regionUrls) - detailCrawl(pageUrls) + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); +// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1" + def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1" + WebDriver driver = new ChromeDriver() + getAllCityRegions(driver, url); } - def regionCrawl(def seedUrl){ - def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk"); - def regionUrls = [] - if(document){ - def eles = JQueryUtils.select(".small-subcondion a", document) - if(eles){ - for(def el : eles){ - def reiongUrl = el.attr("href").trim() - if(reiongUrl.startsWith("//")){ - regionUrls << "http:"+reiongUrl + def getAllCityRegions(driver, url){ + + def provinces = getAllProvince(driver, url) + def citys = ["天津","上海","北京","重庆"] + + for(def province : provinces){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles= JQueryUtils.select(".unfold .J_FilterCity a") + for(def cityEle : cityEles){ + def city = cityEle.text() + if(citys.contains(city)){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + doc = Jsoup.parse(driver.getPageSource()); + + driver.findElement(By.cssSelector(".J_FilterCity a")).click() + doc = Jsoup.parse(driver.getPageSource()); + def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a") + for(def r : regionEles){ + def m = [:] + m.province = province + m.city = city + m.region = r.text() + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } + }else{ + def m = [:] + m.province = province + m.city = city + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } } } - return regionUrls; } - def pageUrlCrawl(regionUrls){ + def crawlLinks(def m, def driver){ + def stop = false def urls = [] - for(def url : regionUrls){ - def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk"); - if(document){ - def pages = JQueryUtils.selectText(".page-total",document) - if(pages){ - pages = pages.toInteger() - for(def page=1;page -1) { + result.address = addr.substring(0, addr.indexOf("室") + 1) + } else { + result.address = addr + } + + if (result.city) { + if (result.address.contains(city + "市")) { + result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1) + } else { + result.address = result.address.substring(result.address.indexOf(city) + city.length()) + } + } + + + if (result.address.indexOf("】") > -1) { + result.address = result.address.substring(result.address.indexOf("】") + 1) + } + + + if (result.city) { + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + } + if (statusDesc) { + def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1) + result.status = DataUtil.getPaiMaiType(status) + result.statusDesc = statusDesc + } + + def statusEle = JQueryUtils.select(".status-tip", doc) + if (statusEle) { + def sEle = JQueryUtils.select(".status-ing", doc) + if (sEle) { + result.status = "进行中" + } else { + result.status = "即将开始" + } + } + def curPrice = JQueryUtils.selectText(["#sf-price"], doc) + if (curPrice) { + if (curPrice.contains("当前价")) { + curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (curPrice) { + curPrice = curPrice.replace(",", "") + if (curPrice.isInteger()) { + result.curPrice = curPrice.toInteger() + } else { + result.curPrice = (int) (curPrice.toDouble()) + } + } + } else if (curPrice.contains("成交价")) { + def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (tradePrice) { + tradePrice = tradePrice.replace(",", "") + if (tradePrice.isInteger()) { + result.tradePrice = tradePrice.toInteger() + } else { + result.tradePrice = (int) (tradePrice.toDouble()) + } + + } + } else if (curPrice.contains("起拍价")) { + def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (startPrice) { + startPrice = startPrice.replace(",", "") + if (startPrice.isInteger()) { + result.startPrice = startPrice.toInteger() + } else { + result.startPrice = (int) (startPrice.toDouble()) + } + + } + } + } + + def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc) + if (finishStr) { + if (finishStr.contains("结束时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } else if (finishStr.contains("成交时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } + } + + def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc) + if (enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc) + if (callCount) { + result.callCount = callCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#J_Looker"], doc) + if (lookCount) { + result.lookCount = lookCount.toInteger() + } + def tds = JQueryUtils.select(["#J_HoverShow td"], doc) + if (tds && tds.size() > 0) { + for (def td : tds) { + def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "") + if (str) { + if (str.contains("变卖价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffPrice = p.toInteger() + } else { + result.sellOffPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("加价幅度")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.addPrice = p.toInteger() + } else { + result.addPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("类型")) { + + result.type = td.select(".pay-type").text() + + } else if (str.contains("保证金")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.depositPrice = p.toInteger() + } else { + result.depositPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("变卖周期")) { + + result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("竞价周期")) { + + result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("优先购买权")) { + + result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("变卖预缴款")) { + def str1 = td.select("span:eq(1) .J_Price").text() + def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffAdvancePay = p.toInteger() + } else { + result.sellOffAdvancePay = (int) (p.toDouble()) + } + } + + } else if (str.contains("评估价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.evaluationPrice = p.toInteger() + } else { + result.evaluationPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("延时周期")) { + + result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("起拍价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.startPrice = p.toInteger() + } else { + result.startPrice = (int) (p.toDouble()) + } + } + } + } + } + } + + result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc) + + def trs = JQueryUtils.select(["#J_desc table tr"], doc) + if (trs) { + def detailMap = [:] + for (def tr : trs) { + def d_tds = tr.select("td") + if (d_tds && d_tds.size() >= 2) { + if (d_tds.size() == 3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } else { + if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + + if (detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + + if (!result.address) { + result.address = detailMap.get("拍品名称") + if (!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs = JQueryUtils.select([".slide-bigpic img"], doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + + } + result.imgUrls = imgLinks + } + } + + //面积 + def article = JQueryUtils.selectText(["#J_ItemNotice"], doc) + if (article) { + def area = DataUtil.getAreaNumber(article, 5000) + if (!area) { + article = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (!area) { + article = JQueryUtils.selectText(["body"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (area) { + result.area = area.toDouble() + if (result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble()) } } } - } - return urls; - } - def detailCrawl(pageUrls){ - if(pageUrls){ - for(def pageUrl : pageUrls){ - def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk"); - if(document){ - def pages = JQueryUtils.select(".page-total", document) + + def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (s1) { + def dateRangs = DataUtil.getDateRange(s1) + if (dateRangs) { + for (def j = 0; j < dateRangs.size(); j++) { + def date = dateRangs[j] + def pattern = null + if (date.endsWith("日时") || date.endsWith("日")) { + pattern = "yyyy年MM月dd日" + } else { + + def ind = date.indexOf("日") + if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) { + date = date + "时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if (j == 0) { + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + } else { + if (!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + + if (!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) } } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + def estateType = DataUtil.findEstateType(s) + if (estateType) { + result.matchEstateType = estateType + } + + result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() + result.url = url + result.mDate = new Date() + results << result } + return results; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index 1ce1c48c7..bd59e65c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -1,20 +1,38 @@ package com.fangjia.script.loudong +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong class CqfdcMatch { + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + static main(args) { def task = new CqfdcMatch(); // task.run() // task.stat() - task.export2() +// task.export2() +// task.matchLoudong() + + task.addLoudong() } def export(){ def q = new BasicDBObject() @@ -139,6 +157,29 @@ class CqfdcMatch { return rets; } + def findAllLoundongByHouseNum(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000) + def rets = [] + def map = [:] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num + if(d.unit_num){ + key = key +"##"+d.unit_num + } + def detailMap= map.get(key) + if(!detailMap){ + detailMap = [:] + map.put(key,detailMap) + } + + if(detailMap.height){ + + } + } + return rets; + } + def stat(){ def q = new BasicDBObject() @@ -168,6 +209,91 @@ class CqfdcMatch { } + def matchLoudong(){ + def names = findAllLoudong("重庆") + print names.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(names.size()) + + def retMap = new ConcurrentHashMap() + for(def i=0;i + def arrs = key.split("#") + + def q = new BasicDBObject("city", arrs[0]) + q.put('region', arrs[1]); + q.put('name', arrs[2]); + + def udata = new BasicDBObject() + udata.put("district_id", m.id.toString()) + udata.put("name_in_seaweed", m.name) + udata.put("district_no", m.sid) + HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true); + index++ + println index + } + } + + def findAllLoudong(city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district_id",new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def names = new CopyOnWriteArrayList() + for(def db : dbcursor){ + def key = db.city +"#"+db.region+"#"+db.name + if(!names.contains(key)){ + names << key + } + } + return names; + } + + def addLoudong(){ + + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ace3084c1..ee0977a28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -192,7 +192,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] + def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index d37680fac..fe433f087 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -245,6 +245,48 @@ public class CommonUtils { } + + static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) { + + + calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate) + if (estateType && estateType.contains("别墅")) { + estateType = "别墅" + } else if (estateType) { + estateType = "小区" + } + + DBObject cmd = new BasicDBObject() + cmd.put("city", city) + cmd.put("region", region) + cmd.put("name", name) + cmd.put('estate_type', estateType) + cmd.put("calc_date", calcDate) + cmd.put("calc_type", 0) + println(cmd) + DBObject data = new BasicDBObject() + data.put("category", "square_rent") + data.put("priority", 3) + data.put("status", 0) + data.put("remark", 'week') + data.put("c_date", new Date()) + + + if(isTest){ + data.put("ip", "192.168.0.231") + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + }else{ + data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)]) + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + } + if(sync){ + while (COMMAND_COLL.count(cmd) != 0){ + Thread.sleep(1000) + } + } + + } + /** * 重算小区均价 * @param city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 3b83880cd..69a75d454 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -314,6 +314,43 @@ public class DbUtils{ } return districts + } + + /** + * 查询附近一共里内小区的数量 + */ + def static findAroundDistrictByRadius(BasicDBObject district, def radius){ + def districts = [] + if(hasGeo(district)){ + def query = new BasicDBObject() + query.append("cat", "district") + query.append("city", district.city) + query.append("visible", 0) + DBObject sdq = new BasicDBObject(); + def lat = district.lat / 10000000000.0 + def lng = district.lng / 10000000000.0 + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radius / 1000.0)); + query.append('$sd', sdq) + + def keys = new BasicDBObject() + keys.append("city", 1) + keys.append("region", 1) + keys.append("name", 1) + keys.append("lat", 1) + keys.append("lng", 1) + keys.append("lat2", 1) + keys.append("lng2", 1) + keys.append("estate_type2", 1) + DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo-> + if(!district._id || dbo._id != district._id){ + districts << dbo + } + } + } + return districts + } /** * 查询小区均价 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 3810aa9f3..b7fe9aa99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220501", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 From 3e2f58f80d0cf4f5559579dee5003ac4d549e7f4 Mon Sep 17 00:00:00 2001 From: xuhe Date: Sat, 7 May 2022 19:14:40 +0800 Subject: [PATCH 18/83] =?UTF-8?q?=E4=BC=B0=E5=80=BC=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=80=BB=E6=A5=BC=E5=B1=82=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/mongo/model/bkoff/Loudong.java | 15 ++++++++++++++- .../open/controllers/PropertyController.java | 6 ++++++ .../service/mongo/bkoff/LoudongService.java | 2 ++ .../mongo/bkoff/LoudongServiceImpl.java | 19 +++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java index 712b5fc7d..12cb8ca9b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java @@ -101,7 +101,11 @@ public class Loudong implements MongoEntity { * seaweed中的小区名 */ private String nameInSeaweed; - + /** + * seaweed中的小区id + */ + private String idInSeaweed; + @Column(name = "remark") public String getRemark() { return remark; @@ -402,4 +406,13 @@ public class Loudong implements MongoEntity { public void setNameInSeaweed(String nameInSeaweed) { this.nameInSeaweed = nameInSeaweed; } + + @Column(name = "id_in_seaweed") + public String getIdInSeaweed() { + return idInSeaweed; + } + + public void setIdInSeaweed(String idInSeaweed) { + this.idInSeaweed = idInSeaweed; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index eb9325a57..db58913eb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -948,6 +948,12 @@ public class PropertyController { return mav; } + if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){ + Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber()); + if(loudong != null && loudong.getFloors() >= model.getFloor()){ + model.setMaxFloor(loudong.getFloors()); + } + } model.setRegion(district.getRegion()); model.setDistrictName(district.getName()); if (StringUtils.isBlank(model.getPropertyType())) { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java index a38fdab95..087123cbb 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java @@ -41,4 +41,6 @@ public interface LoudongService { void save(Map data); List findAllSources(); + + Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java index 57e9c6bad..1efc1c617 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java @@ -318,4 +318,23 @@ public class LoudongServiceImpl implements LoudongService { } return froms; } + + @Override + public Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber){ + if(StringUtils.isBlank(idInSeaweed) || StringUtils.isBlank(houseNumber)){ + return null; + } + List params = Lists.newArrayList(); + params.add(new QueryParam("idInSeaweed", idInSeaweed)); + params.add(new QueryParam("houseNumber", houseNumber)); + if(StringUtils.isNotBlank(unitNumber)) { + params.add(new QueryParam("unitNumber", unitNumber)); + } + params.add(new QueryParam("floors", QueryParam.GREATER_THAN_OPERATOR, 0)); + params.add(new QueryParam("floors", QueryParam.LESS_THAN_OPERATOR, 60)); + params.add(new QueryParam("status", QueryParam.NOT_EQUAL_OPERATOR, 0)); + + List sortParams = ImmutableList.of(new SortParam("floors", false)); + return loudongDao.findFirstByParams(params, sortParams); + } } From c066c22dc34c359dd8884039eeecd0ef0f6654dc Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 8 May 2022 16:23:17 +0800 Subject: [PATCH 19/83] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E7=9A=84ip=E5=92=8Cheader=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/requestController.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java new file mode 100644 index 000000000..d23a312bb --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java @@ -0,0 +1,36 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.AnalyticsUtil; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; + +import javax.servlet.http.HttpServletRequest; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +public class requestController { + + /** + * 获取请求的request和IP信息 + * @param request + * @return + */ + public Object requestAction(HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + String ip = AnalyticsUtil.getIpAddress(request); + Map map = new HashMap(); + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String key = (String) headerNames.nextElement(); + String value = request.getHeader(key); + map.put(key, value); + } + map.put("ip", ip); + mav.setModel(BasicModel.buildOKModel().setResult(map)); + return mav; + } +} From b8570b104e1209209880d6ed81d3e095a558cad2 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 11 May 2022 12:39:35 +0800 Subject: [PATCH 20/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9D=BF=E5=9D=97?= =?UTF-8?q?=E6=89=80=E5=B1=9E=E5=8C=BA=E5=9F=9F=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/geo/BlockRegion.groovy | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy new file mode 100644 index 000000000..bdd0d1110 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.geo + +import com.fangjia.script.gj.location.Point +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.xh.InsidePolygon +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017) + +collection.find(new BasicDBObject()).forEach({ obj -> + if (obj.type == 'block') { + if (obj.region instanceof String) { + return; + } + BasicDBList reigon = (BasicDBList)obj.region + int count = 0 + List info = new ArrayList<>() + for (String s : reigon) { + info.add(s) + } + if (info.size() > 1) { + boolean find = false + for (String s : info) { + if (find) { + break + } + collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 -> + String border = obj2.border + String points = border.replaceAll(",", "-").replaceAll(";", "-") + List pointList = InsidePolygon.createPoint(points); + com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat) + if (InsidePolygon.aboutZone(point, pointList)) { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", obj2.name))); + find = true; + } + }) + } + } else { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", info.get(0)))); + } + } +}) \ No newline at end of file From 2a70fb989cea97eb0d6285e52d0b95802aa18179 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 11 May 2022 13:31:49 +0800 Subject: [PATCH 21/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/mongo/model/DistrictFacilitys.java | 9 ++++ .../dal/mongo/model/bkoff/HouseLoudong.java | 1 + .../dal/mongo/model/bkoff/Loudong.java | 13 +++++ .../open/controllers/DistrictController.java | 53 +++++++++++++------ .../open/controllers/PropertyController.java | 35 +++++++++++- .../com/fangjia/open/dto/DistrictInfoDto.java | 9 ++++ .../open/model/PropertyEvaluationModel.java | 21 ++++++++ .../mongo/bkoff/HouseLoudongService.java | 2 + .../mongo/bkoff/HouseLoudongServiceImpl.java | 16 ++++++ .../service/mongo/bkoff/LoudongService.java | 2 + .../mongo/bkoff/LoudongServiceImpl.java | 17 ++++++ 11 files changed, 160 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java index 36021519c..cc946257b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java @@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity { private String imgUrl3D; private List school; + private List keySchool; @Id @GeneratedValue @@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity { this.school = school; } + @Column(name = "key_school") + public List getKeySchool() { + return keySchool; + } + + public void setKeySchool(List keySchool) { + this.keySchool = keySchool; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java index cf98a3665..d41b400ee 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java @@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{ this.openTotalPrice = openTotalPrice; } + @Column(name = "district_id") public String getDistrictId() { return districtId; } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java index 712b5fc7d..ef04d1c45 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java @@ -101,6 +101,10 @@ public class Loudong implements MongoEntity { * seaweed中的小区名 */ private String nameInSeaweed; + /** + * 小区编号 + */ + private String idInSeaweed; @Column(name = "remark") public String getRemark() { @@ -402,4 +406,13 @@ public class Loudong implements MongoEntity { public void setNameInSeaweed(String nameInSeaweed) { this.nameInSeaweed = nameInSeaweed; } + + @Column(name = "id_in_seaweed") + public String getIdInSeaweed() { + return idInSeaweed; + } + + public void setIdInSeaweed(String idInSeaweed) { + this.idInSeaweed = idInSeaweed; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java index 859a04ef0..395cb8a8d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java @@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest; import com.fangjia.core.RegionData; import com.fangjia.core.util.DateUtil; -import com.fangjia.dal.mongo.model.RegionStat; -import com.fangjia.dal.mongo.model.RegionStatRent; +import com.fangjia.dal.mongo.model.*; import com.fangjia.open.model.AvgPriceQueryModel; import com.fangjia.open.model.BasicModel; import com.fangjia.service.datastore.RegionStatService; +import com.fangjia.service.district.DistrictFacilitysService; import com.fangjia.service.district.DistrictStatRentService; import com.fangjia.service.district.RegionStatRentService; import com.fangjia.service.mongo.fangjia.AverageAnalyService; @@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.FangjiaServerAreaContextHolder; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; -import com.fangjia.dal.mongo.model.DistrictHouse; -import com.fangjia.dal.mongo.model.DistrictStat; import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.ApiUser; @@ -93,6 +91,7 @@ public class DistrictController { private RegionBlockService regionBlockService; private RegionStatService regionStatService; private RegionStatRentService regionStatRentService; + private DistrictFacilitysService districtFacilitysService; public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea", @@ -444,19 +443,19 @@ public class DistrictController { * * @return */ - public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, - @Param(name = "name") String name, HttpServletRequest request) { - RestfulModelAndView mav = new RestfulModelAndView(); - mav.setViewType("rest"); - if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { - mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); - return mav; - } else { - DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name)); - if (null == districtBo) { - mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件")); + public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, + @Param(name = "name") String name, HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); return mav; - } + } else { + DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name)); + if (null == districtBo) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件")); + return mav; + } districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey())); // 去除小区描述中的html标签 districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription())); @@ -1146,6 +1145,25 @@ public class DistrictController { ret.put("RESULT", 200); return mav; } + + public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, + @Param(name = "name") String name, HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); + return mav; + } else { + District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district"); + if (null == districtBo) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在")); + return mav; + } + DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo); + mav.setModel(BasicModel.buildOKModel().setResult(facilitys)); + } + return mav; + } @@ -1230,4 +1248,7 @@ public class DistrictController { this.regionStatRentService = regionStatRentService; } + public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) { + this.districtFacilitysService = districtFacilitysService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index f6e9855f0..29b964d55 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -740,7 +740,7 @@ public class PropertyController { } else { districtInfoDto.setBuildingType(""); } - districtInfoDto.setAlias(district.getAlias()); + districtInfoDto.setAlias(district.getMappingAlias()); districtInfoDto.setAddressAlias(district.getAddrAlias()); districtInfoDto.setCommunity(district.getStreetCommunity()); districtInfoDto.setCommercialArea(district.getBusinessQuarter()); @@ -792,6 +792,7 @@ public class PropertyController { districtInfoDto.setDescription(district.getDescription()); districtInfoDto.setSalesOfficeAdd(district.getSellAddress()); districtInfoDto.setSalesPhoneNumber(district.getMobile()); + districtInfoDto.setTownship(district.getTownship()); if (district.getLoudongCount() != null && district.getLoudongCount() > 0) { districtInfoDto.setBuildingCount(district.getLoudongCount()); } @@ -1116,7 +1117,7 @@ public class PropertyController { model.setCalDate(calDate); } /* 先到CPDB库中查找小区 */ - District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName()); + District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName()); if(district == null){ //防止地址模糊,名称详细的情况 district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress()); @@ -5412,6 +5413,36 @@ public class PropertyController { } } + public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + + if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){ + mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项")); + return mav; + } + + List houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber()); + mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs)); + return mav; + } + + + public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + + if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){ + mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项")); + return mav; + } + + + List loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber()); + mav.setModel(BasicModel.buildOKModel().setResult(loudongs)); + return mav; + } + public void setHouseLoudongService(HouseLoudongService houseLoudongService) { this.houseLoudongService = houseLoudongService; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java index 1ab67359e..d938d3a8b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java @@ -99,6 +99,7 @@ public class DistrictInfoDto { private String salesPhoneNumber; private Integer buildingCount; private Integer finishYear; + private String township; public String getDistrict() { @@ -818,4 +819,12 @@ public class DistrictInfoDto { public void setFinishYear(Integer finishYear) { this.finishYear = finishYear; } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 5953fc3a3..859658742 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { * 楼号 */ private String unitNumber; + /** + * 新楼号 + */ + private String houseNumber; + private String id; /** * 室号 */ @@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } + + public String getHouseNumber() { + return houseNumber; + } + + public void setHouseNumber(String houseNumber) { + this.houseNumber = houseNumber; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java index 3f6ad7f39..236cf00c0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java @@ -94,4 +94,6 @@ public interface HouseLoudongService { public List findAll(List queryParams, Pager pager); List findAllSources(); + + public List findAll(String id, String houseNum, String unitNum, String roomNum); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java index 70286758b..9fe6af3ef 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java @@ -297,6 +297,22 @@ public class HouseLoudongServiceImpl implements HouseLoudongService { return froms; } + @Override + public List findAll(String id, String houseNum, String unitNum, String roomNum) { + List params = new ArrayList(); + params.add(new QueryParam("districtId", id)); + if (StringUtils.isNotBlank(houseNum)) { + params.add(new QueryParam("houseNum", houseNum)); + } + if (StringUtils.isNotBlank(unitNum)) { + params.add(new QueryParam("houseNumUnit", unitNum)); + } + if (StringUtils.isNotBlank(roomNum)) { + params.add(new QueryParam("roomNum", roomNum)); + } + return houseLoudongBkoffDao.findAllByParams(params); + } + @Override public List queryHouseLoudongList(String city, String districtName, String roomNum, Pager pager) { List params = new ArrayList(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java index a38fdab95..682b2aaca 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java @@ -23,6 +23,8 @@ public interface LoudongService { List findAll(String city, String region, String name, String houseNumber, String unitNumber, int limit); + List findAll(String id, String houseNumber, String unitNumber); + Map findAll(List query, String city); Loudong findById(String id); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java index 57e9c6bad..05e2f9f3e 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java @@ -148,6 +148,23 @@ public class LoudongServiceImpl implements LoudongService { return loudongDao.findAllByParams(params,null, 0 , limit, ADDRESS_VERIFY_FIELDS); } + @Override + public List findAll(String id, String houseNumber, String unitNumber) { + List params = new ArrayList(); + if(StringUtils.isNotBlank(id)) { + params.add(new QueryParam("idInSeaweed", id)); + } + + if(StringUtils.isNotBlank(houseNumber)) { + params.add(new QueryParam("houseNumber", houseNumber)); + } + if(StringUtils.isNotBlank(unitNumber)) { + params.add(new QueryParam("unitNumber", unitNumber)); + } + + return loudongDao.findAllByParams(params); + } + @Override public Loudong findById(String id) { return loudongDao.findById(id); From 816616484f61be639ccf7c20c8d993f4d8717b20 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 12 May 2022 00:11:57 +0800 Subject: [PATCH 22/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/mongo/dao/BsEvaluationDao.java | 7 + .../dal/mongo/dao/BsEvaluationDaoImpl.java | 8 + .../fangjia/dal/mongo/model/BsEvaluation.java | 184 ++++++++++++++++++ .../dal/mongo/model/bkoff/DealPrice.java | 41 ++++ .../dal/seawater/dao/SeawaterDaoImpl.java | 7 + .../fangjia/dal/seawater/model/Seawater.java | 20 ++ .../open/controllers/AuctionController.java | 25 ++- .../open/controllers/DealPriceController.java | 38 ++++ .../controllers/EvaluationController.java | 62 +++++- .../open/controllers/HouseController.java | 58 ++++++ .../open/controllers/PropertyController.java | 1 + .../com/fangjia/open/model/AuctionModel.java | 9 + .../fangjia/open/model/DealPriceModel.java | 39 ++++ .../fangjia/open/model/EvaluationModel.java | 36 ++++ .../fangjia/open/model/HouseSearchModel.java | 12 +- .../com/fangjia/open/model/SearchModel.java | 18 ++ .../datastore/BsEvaluationService.java | 11 ++ .../datastore/BsEvaluationServiceImpl.java | 44 +++++ .../mongo/fangjia/DealPriceService.java | 2 + .../mongo/fangjia/DealPriceServiceImpl.java | 23 +++ 20 files changed, 635 insertions(+), 10 deletions(-) create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java new file mode 100644 index 000000000..5f895886b --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java @@ -0,0 +1,7 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.BsEvaluation; + +public interface BsEvaluationDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java new file mode 100644 index 000000000..e4d7fc080 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java @@ -0,0 +1,8 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.model.BsEvaluation; + +public class BsEvaluationDaoImpl extends MongoDaoImpl implements + BsEvaluationDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java new file mode 100644 index 000000000..d3b1b28f0 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java @@ -0,0 +1,184 @@ +package com.fangjia.dal.mongo.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; +import java.util.Date; + +@Entity +@Table(name = "business.evaluation") +public class BsEvaluation implements MongoEntity { + private String id; + private String fullAddress; + private String commid; + private String cityName; + private String distName; + private String commName; + private String address; + private Double buildArea; + private Integer floor; + private Integer floorcnt; + private Integer roomcnt; + private Integer livingroomcnt; + private String houseaspect;//南 + private String fitment; + private Date cDate; + private Integer unitPrice; + private Integer totalPrice; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + @Column(name = "_id") + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "full_address") + public String getFullAddress() { + return fullAddress; + } + + public void setFullAddress(String fullAddress) { + this.fullAddress = fullAddress; + } + + @Column(name = "commid") + public String getCommid() { + return commid; + } + + public void setCommid(String commid) { + this.commid = commid; + } + + @Column(name = "cityname") + public String getCityName() { + return cityName; + } + + public void setCityName(String cityName) { + this.cityName = cityName; + } + + @Column(name = "distname") + public String getDistName() { + return distName; + } + + public void setDistName(String distName) { + this.distName = distName; + } + + @Column(name = "commname") + public String getCommName() { + return commName; + } + + public void setCommName(String commName) { + this.commName = commName; + } + + @Column(name = "address") + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + @Column(name = "buildarea") + public Double getBuildArea() { + return buildArea; + } + + public void setBuildArea(Double buildArea) { + this.buildArea = buildArea; + } + + @Column(name = "floor") + public Integer getFloor() { + return floor; + } + + public void setFloor(Integer floor) { + this.floor = floor; + } + + @Column(name = "floorcnt") + public Integer getFloorcnt() { + return floorcnt; + } + + public void setFloorcnt(Integer floorcnt) { + this.floorcnt = floorcnt; + } + + @Column(name = "roomcnt") + public Integer getRoomcnt() { + return roomcnt; + } + + public void setRoomcnt(Integer roomcnt) { + this.roomcnt = roomcnt; + } + + @Column(name = "livingroomcnt") + public Integer getLivingroomcnt() { + return livingroomcnt; + } + + public void setLivingroomcnt(Integer livingroomcnt) { + this.livingroomcnt = livingroomcnt; + } + + @Column(name = "houseaspect") + public String getHouseaspect() { + return houseaspect; + } + + public void setHouseaspect(String houseaspect) { + this.houseaspect = houseaspect; + } + + @Column(name = "fitment") + public String getFitment() { + return fitment; + } + + public void setFitment(String fitment) { + this.fitment = fitment; + } + + @Column(name = "c_date") + public Date getcDate() { + return cDate; + } + + public void setcDate(Date cDate) { + this.cDate = cDate; + } + + @Column(name = "unitprice") + public Integer getUnitPrice() { + return unitPrice; + } + + public void setUnitPrice(Integer unitPrice) { + this.unitPrice = unitPrice; + } + + @Column(name = "totalprice") + public Integer getTotalPrice() { + return totalPrice; + } + + public void setTotalPrice(Integer totalPrice) { + this.totalPrice = totalPrice; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java index a4ababe8e..a46cd7de0 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java @@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity { private Integer floor;//当前楼层 private Long score; + private String matchId; + private String matchCity; + private String matchRegion; + private String matchName; + @Id @GeneratedValue @Column(name = "_id", unique = true, nullable = false) @@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity { } return true; } + + @Column(name = "match_id") + public String getMatchId() { + return matchId; + } + + public void setMatchId(String matchId) { + this.matchId = matchId; + } + + @Column(name = "match_city") + public String getMatchCity() { + return matchCity; + } + + public void setMatchCity(String matchCity) { + this.matchCity = matchCity; + } + + @Column(name = "match_region") + public String getMatchRegion() { + return matchRegion; + } + + public void setMatchRegion(String matchRegion) { + this.matchRegion = matchRegion; + } + + @Column(name = "match_name") + public String getMatchName() { + return matchName; + } + + public void setMatchName(String matchName) { + this.matchName = matchName; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index e71226a4a..203b6818b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -67,6 +67,13 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i must.add(addTerms("district_name", alias)); must.add(addMatch("cat", "sell")); must.add(addRange("avg_price", minPrice, maxPrice)); + + JSONObject term = new JSONObject(); + JSONObject match = new JSONObject(); + match.put("p_date", 20210501); + term.put("match", match); + must.add(term); + bool.put("must", must); query.put("bool", bool); reqBody.put("query", query); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index c761ba76a..48005c87b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -58,6 +58,8 @@ public class Seawater implements SeawaterEntity { private String url; private String root; private Integer avgPrice; + private String purpose; + private String estateType; @Id @Column(name = "_id") @@ -285,4 +287,22 @@ public class Seawater implements SeawaterEntity { public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } + + @Column(name = "purpose") + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + @Column(name = "estate_type") + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index 13e59faa4..ed4a73f2d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -23,13 +23,13 @@ import java.util.regex.Pattern; /** * 法拍数据提供接口 */ -@Authority(role = Role.USER) +//@Authority(role = Role.USER) public class AuctionController { private PaiMaiService paiMaiService; private DistrictNewPriceService districtNewPriceService; - private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"}; + private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType","type","count"}; public Object housesAction(AuctionModel model){ RestfulModelAndView mav = new RestfulModelAndView(); @@ -187,6 +187,27 @@ public class AuctionController { return mav; } + + public Object searchPaimaisAction(AuctionModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + List params = new ArrayList(); + if(StringUtils.isNotBlank(model.getCity())){ + params.add(new QueryParam("matchCity", model.getCity())); + } + if(StringUtils.isNotBlank(model.getDistrict())){ + params.add(new QueryParam("matchRegion", model.getDistrict())); + } + + if(StringUtils.isNotBlank(model.getName())){ + params.add(new QueryParam("matchName", model.getName())); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("startDate", false)); + List paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS); + mav.setModel(BasicModel.buildOKModel().setResult(paimais)); + return mav; + } + public void setPaiMaiService(PaiMaiService paiMaiService) { this.paiMaiService = paiMaiService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java index 8abece9e2..ab916308a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java @@ -15,8 +15,10 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.Metro; import com.fangjia.open.model.BasicModel; +import com.fangjia.service.bo.DistrictStatBo; import org.apache.commons.lang.StringUtils; import com.fangjia.core.util.DateUtil; @@ -211,6 +213,42 @@ public class DealPriceController { mv.setModel(resMap); return mv; } + + public Object searchListAction(final DealPriceModel model) { + RestfulModelAndView mv = new RestfulModelAndView(); + mv.setViewType("rest"); + + if (StringUtils.isBlank(model.getId())) { + mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空")); + return mv; + } + District district = districtService.findDistrictById(model.getId()); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + if(district == null){ + mv.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mv; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + Date startDate = null; + Date endDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + } + if(StringUtils.isNotBlank(model.getEndDate())){ + endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd"); + } + + List dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit()); + mv.setModel(BasicModel.buildOKModel().setResult(dealPrices)); + return mv; + } private List findDealPricesByName(String city,String region ,String name){ Calendar calendar = Calendar.getInstance(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java index 67e7b084c..7ed1a83f4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java @@ -10,9 +10,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fangjia.dal.mongo.model.BsEvaluation; +import com.fangjia.dal.mongo.model.bkoff.DealPrice; import com.fangjia.service.bo.AddrEvaResult; +import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.bo.EvaluationBo; import com.fangjia.service.bo.HouseScorer; +import com.fangjia.service.datastore.*; +import com.fangjia.service.district.DistrictStatService; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.api.AutoCompleteService; import com.fangjia.service.api.KnowbaseService; import com.fangjia.service.common.SellSimilar; -import com.fangjia.service.datastore.BusinessEstateService; -import com.fangjia.service.datastore.DistrictService; -import com.fangjia.service.datastore.EvaluationReportService; -import com.fangjia.service.datastore.EvaluationService; -import com.fangjia.service.datastore.HouseService; import com.fangjia.service.sql.ApiUserService; import com.fangjia.service.sql.UserService; import com.fangjia.service.web.AddressService; @@ -94,6 +94,8 @@ public class EvaluationController { private ApiUserService apiUserService; private UserService userService; private AddressService addressService; + private DistrictStatService districtStatService; + private BsEvaluationService bsEvaluationService; private final static String AK = OpenConfig.getConfig().getBaiduAk(); public Object lngLatEvaluateAction(EvaluationModel model) { @@ -1005,6 +1007,44 @@ public class EvaluationController { return mav; } + public Object bsListAction(EvaluationModel model) { + RestfulModelAndView mv = new RestfulModelAndView(); + mv.setViewType("rest"); + + if (StringUtils.isBlank(model.getId())) { + mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空")); + return mv; + } + District district = districtService.findDistrictById(model.getId()); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + if(district == null){ + mv.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mv; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + Date startDate = null; + Date endDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + } + if(StringUtils.isNotBlank(model.getEndDate())){ + endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd"); + } + + List dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit()); + mv.setModel(BasicModel.buildOKModel().setResult(dealPrices)); + return mv; + } + + + public void setEvaluationService(EvaluationService evaluationService) { this.evaluationService = evaluationService; } @@ -1017,6 +1057,18 @@ public class EvaluationController { this.knowbaseService = knowbaseService; } + + + public void setDistrictStatService(DistrictStatService districtStatService) { + this.districtStatService = districtStatService; + } + + + + public void setBsEvaluationService(BsEvaluationService bsEvaluationService) { + this.bsEvaluationService = bsEvaluationService; + } + public void setAutoCompleteService(AutoCompleteService autoCompleteService) { this.autoCompleteService = autoCompleteService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index ebb6c9025..bdf38d8b0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys; import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap; import com.fangjia.dal.mongo.model.Metro; import com.fangjia.dal.mongo.model.StatsIndex; +import com.fangjia.dal.seawater.dao.SeawaterDao; +import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.ApiUser; import com.fangjia.datastore.driver.FacetResult; @@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery; import com.fangjia.open.ws.query.ShopCollateralSampleQuery; import com.fangjia.service.address.AddressParser; import com.fangjia.service.bo.Collateral; +import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.bo.Sample; import com.fangjia.service.bo.ShopSample; import com.fangjia.service.datastore.CzbankSampleService; @@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean; import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean; import com.fangjia.service.district.DistrictFacilitysService; import com.fangjia.service.district.DistrictStatService; +import com.fangjia.service.district.SqmetreRentService; import com.fangjia.service.menu.Item; import com.fangjia.service.menu.MenuCategoryEnum; import com.fangjia.service.menu.Value; import com.fangjia.service.mongo.fangjia.DealPriceService; import com.fangjia.service.mongo.fangjia.MetroService; +import com.fangjia.service.mongo.fangjia.SeawaterService; import com.fangjia.service.mongo.fangjia.StatsIndexService; import com.fangjia.service.open.FangjiaOfficialAreaService; import com.fangjia.service.open.HousePojo; @@ -86,6 +91,9 @@ public class HouseController { private CzbankSampleService czbankSampleService; private DistrictService districtService; private FangjiaOfficialAreaService fangjiaOfficialAreaService; + + private SeawaterService seawaterService; + private SeawaterDao seawaterDao; private List PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02"); @@ -399,6 +407,49 @@ public class HouseController { } } + public Object listAction(final HouseSearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) { + mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项")); + return mav; + } + District district = null; + DistrictKey districtKey = null; + if(StringUtils.isNotBlank(model.getName())){ + districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName()); + district = districtService.findDistrict(districtKey, "district"); + }else{ + district = districtService.findDistrictById(model.getId()); + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } + + if(district == null){ + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + List names = new ArrayList(); + names.add(district.getName()); + if(district.getAlias() != null){ + for(String alia : district.getAlias()){ + if(StringUtils.isNotBlank(alia) && !names.contains(alia)){ + names.add(alia); + } + } + } + List houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater"); + //List houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize()); + mav.setModel(BasicModel.buildOKModel().setResult(houses)); + return mav; + } + String getPicUrl(String url, List urls) { if (!urls.contains(url)) { urls.add(url); @@ -832,6 +883,13 @@ public class HouseController { return ret; } + public void setSeawaterService(SeawaterService seawaterService) { + this.seawaterService = seawaterService; + } + + public void setSeawaterDao(SeawaterDao seawaterDao) { + this.seawaterDao = seawaterDao; + } private boolean checkExists(FacetQueryResult houseBoList, HousePojo house) { for (HousePojo bo : houseBoList) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 0d9ca8a80..934ca4cdb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -665,6 +665,7 @@ public class PropertyController { return mav; } String userId = FangjiaApiUtil.parseUserId(model.getToken()); + District district = null; if (StringUtils.isNotBlank(model.getId())){ if(ObjectId.isValid(model.getId())){ diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java index a4d47a4e6..a12e09a99 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java @@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{ private String city; private String district; private String address; + private String name; private Date minStartTime; private Date maxStartTime; private Integer minStartPrice; @@ -121,4 +122,12 @@ public class AuctionModel extends FangjiaApiModel{ public void setStatus(Integer status) { this.status = status; } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java index 0d51d0c46..35620322f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java @@ -1,6 +1,9 @@ package com.fangjia.open.model; +import java.util.Date; + public class DealPriceModel extends FangjiaApiModel{ + private String id; private String city; @@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{ private Double size; private Integer max; + private Integer limit = 10; private String decoration; private boolean convert; + + private String startDate; + private String endDate; public String getCity() { @@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{ public void setConvert(boolean convert) { this.convert = convert; } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java index d5b8f42ed..71130e7bb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java @@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel { private Date buildDate; private String sid;//微信小程序中的sid private String code;//微信小程序中的code + private String id; + private String startDate; + private String endDate; + private Integer limit = 10; + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } public String getRegion() { return region; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java index d9f4812d8..63a85c4b3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java @@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean; import com.fangjia.service.query.QueryBean; public class HouseSearchModel extends SearchModel { + private String id; private String kind; private String estateType; private String buildingType; @@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel { public void setShowDetail(Boolean showDetail) { this.showDetail = showDetail; } - - - + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java index 813f42f6c..8f1311061 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java @@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel { protected Integer totalPriceEnd; protected Integer avgPriceStart; protected Integer avgPriceEnd; + protected Integer startTime; + protected Integer endTime; protected Integer areaStart; protected Integer areaEnd; protected Integer room; @@ -358,4 +360,20 @@ public abstract class SearchModel extends FangjiaApiModel { public void setTotalPrice(Integer totalPrice) { this.totalPrice = totalPrice; } + + public Integer getStartTime() { + return startTime; + } + + public void setStartTime(Integer startTime) { + this.startTime = startTime; + } + + public Integer getEndTime() { + return endTime; + } + + public void setEndTime(Integer endTime) { + this.endTime = endTime; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java new file mode 100644 index 000000000..733fcb368 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java @@ -0,0 +1,11 @@ +package com.fangjia.service.datastore; + +import com.fangjia.dal.mongo.model.BsEvaluation; + +import java.util.Date; +import java.util.List; + +public interface BsEvaluationService { + + List getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java new file mode 100644 index 000000000..4d96aaadd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java @@ -0,0 +1,44 @@ +package com.fangjia.service.datastore; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.SortParam; +import com.fangjia.dal.mongo.dao.BsEvaluationDao; +import com.fangjia.dal.mongo.dao.EvaluationDao; +import com.fangjia.dal.mongo.model.BsEvaluation; +import com.fangjia.dal.mongo.model.bkoff.DealPrice; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class BsEvaluationServiceImpl implements BsEvaluationService { + + private BsEvaluationDao bsEvaluationDao; + + public void setBsEvaluationDao(BsEvaluationDao bsEvaluationDao) { + this.bsEvaluationDao = bsEvaluationDao; + } + + @Override + public List getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit) { + List params = new ArrayList(); + params.add(new QueryParam("commid", id)); + if(startDate!=null){ + params.add(new QueryParam("cDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + if(endDate!=null){ + params.add(new QueryParam("cDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate)); + } + + if (minPrice > 0) { + params.add(new QueryParam("unitPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice)); + } + if (maxPrice > 0) { + params.add(new QueryParam("unitPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice)); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("cDate", false)); + List bsEvaluations = bsEvaluationDao.findAllByParams(params, sortParams, limit); + return bsEvaluations; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java index 9061a3288..fbd4dd7ea 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java @@ -82,6 +82,8 @@ public interface DealPriceService { public List getNearlySameDealPrice(DistrictKey key, int minPrice, int maxPrice, int radius, int limit); + public List getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice,Date startDate, Date endDate, int limit); + public List getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit); /** diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java index be0f7261f..dd205bd77 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java @@ -354,6 +354,29 @@ public class DealPriceServiceImpl implements DealPriceService { return houses; } + @Override + public List getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, int limit) { + List params = new ArrayList(); + params.add(new QueryParam("matchId", districtId)); + if(startDate!=null){ + params.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + if(endDate!=null){ + params.add(new QueryParam("tradeDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate)); + } + + if (minPrice > 0) { + params.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice)); + } + if (maxPrice > 0) { + params.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice)); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("tradeDate", false)); + List dealPrices = dealPriceDao.findAllByParams(params, sortParams, limit); + return dealPrices; + } + @Override public List getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit) { List params = new ArrayList(); From a0874980d9056863a5e73032f3a8cca17b0e872c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 12 May 2022 09:29:14 +0800 Subject: [PATCH 23/83] update --- .../chongqing/MatchEvaluateRegionTest.groovy | 25 +- .../script/chongqing/MatchEvaluateTest.groovy | 2 +- .../fangjia/script/chongqing/PocTest.groovy | 510 ++++++++++++++++++ .../DistrictAroundFacilitiesBatch.groovy | 2 +- .../script/gj/paimai/PaimaiTest.groovy | 87 ++- .../script/loudong/CqfdcChromeCrawl.groovy | 12 +- .../fangjia/script/loudong/CqfdcMatch.groovy | 432 ++++++++++++++- .../temp/shenjian/util/SeaweedUtils.groovy | 4 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 56 +- .../TestCrawlTotalCount.groovy | 4 +- .../fangjia/script/util/OpenApiClient.groovy | 171 ++++++ 11 files changed, 1249 insertions(+), 56 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy index bae011454..cd0df4d4c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -5,6 +5,7 @@ import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.paimai.DataUtil import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.CommonUtils @@ -22,16 +23,19 @@ class MatchEvaluateRegionTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def total = 1500 + static main(args) { def task = new MatchEvaluateRegionTest() task.run()//小区匹配 task.run1()//地图匹配 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 task.addPrice() + task.addArroundPrice() task.addAuthor() } def run(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) @@ -51,7 +55,6 @@ class MatchEvaluateRegionTest { } def run1(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) @@ -71,7 +74,6 @@ class MatchEvaluateRegionTest { } def addPrice(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['L']){ def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) @@ -91,9 +93,22 @@ class MatchEvaluateRegionTest { } + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A'] && !row['K']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['M'] = priceObj.unitPrice + row['N'] = (int) (priceObj.unitPrice * row['C']) + row['Q'] = "周边价格" + } + } + }); + } + def addAuthor(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" - def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def total = 747 def personTotal = authors.size() def avg = Math.ceil(total/personTotal) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy index e0a41a599..936b3b582 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -117,7 +117,7 @@ class MatchEvaluateTest { def addAuthor(){ // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" - def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def personTotal = authors.size() def avg = Math.ceil(total/personTotal) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy new file mode 100644 index 000000000..804d5891a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy @@ -0,0 +1,510 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.util.SourceNameUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CopyOnWriteArrayList + +class PocTest { + + static main(args) { + def task = new PocTest(); + task.run(); + } + + def run(){ +// match("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx") + +// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx") +// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx") +// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") +// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") + createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['P'] && row['C']){ + def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['P'] = jsonObject.get("name") + row['Q'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['Q']){ + def dist = CommonDatautils.findDistrictById(row['Q']) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['E'] = priceObj.avg_price + } + } + } + }) + } + + def standard(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + def jsonObject = OpenApiClient.standard2Action(row['A'], row['C']) + if(jsonObject){ + println jsonObject + row['F'] = jsonObject.get("province") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("township") + row['J'] = jsonObject.get("street") + row['K'] = jsonObject.get("streetNumber") + row['L'] = jsonObject.get("buildingNumber") + row['M'] = jsonObject.get("unitNumber") + row['N'] = jsonObject.get("floor") + row['O'] = jsonObject.get("roomNumber") + } + } + + }) + } + + def addLoudongNum(path){ + + ExcelUtils.multiThreadingModify(path, { row -> + if (row['Q'] && row['L']) { + def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + if (jsonObject && jsonObject.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['S'] = obj.id + row['R'] = obj.houseNumber + } + } + + if(row['O']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['U'] = obj.id + row['T'] = obj.roomNum + } + } + } + } + }) + } + + def createLoudong(path, output){ + def rets = new CopyOnWriteArrayList() + def datas = ExcelUtils.read(path, 0, 1) + rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"] + for(def row : datas){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + if (row['Q'] && row['L']) { + def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) { + def obj = jsonObject.result[0] + data << obj.houseNumber + data << obj.id + data << obj.type// 物业类型 + data << ""//主朝向 + data << obj.floors//地上层数 + data << ""//地下层数 + data << obj.buildYear;//竣工日期 + data << ""//梯 + data << ""//户 + data << obj.households//总户数 + data << ""//景观 + }else{ + data << "" + data << "" + data << ""// 物业类型 + data << ""//主朝向 + data << ""//地上层数 + data << ""//地下层数 + data << ""//竣工日期 + data << ""//梯 + data << ""//户 + data << ""//总户数 + data << ""//景观 + } + + if(row['O']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + + data << obj.roomNum//户号 + data << obj.id//户ID + data << obj.direction//主 + data << obj.sight//景观 + if(obj.roomNum && obj.roomNum.contains("跃")){ + data << "跃式"//户型结构 + }else{ + data << ""//户型结构 + } + + } + } + } + } + rets << data + } + ExcelUtils.write(output, rets, "底层数据-楼栋-户") + } + + def addDetail(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"] + for(def d : datas){ + if(d['Q']){ + def dist = CommonDatautils.findDistrictById(d['Q']) + if(dist){ + def row = [] + def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name) + if(jsonObject) { + + row << d['A'] + row << d['B'] + row << d['C'] + row << jsonObject.get("name")//小区名称 + row << d['Q']//小区ID + if(jsonObject.get("alias")){ + row << jsonObject.get("alias").join(",").replace('"',"")//小区别名 + }else{ + row << "" + } + + row << jsonObject.get("address")//小区地址 + row << jsonObject.get("commercialArea")//板块 + row << jsonObject.get("township")//街道*********** + row << jsonObject.get("avgPrice")//基准价 + row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度 + row << jsonObject.get("finishBuildTime")//竣工时间 + row << jsonObject.get("developer")//开发商名称 + row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率 + row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率 + row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数 + row << jsonObject.get("propertyManagement")//物业公司 + row << jsonObject.get("managementFee")//物业费 + row << jsonObject.get("buildingCount")//住宅幢数 + row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数 + row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比********* + row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积 + + } + + def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name) + if(facilityObj && facilityObj.result){ + def result = facilityObj.result + if(result.keySchool){ + row << '是' + }else { + row << "" + } + + if(result.school){ + row << result.school.join(",").replace('"',"") + }else { + row << "" + } + + def bus = [] + if(result.bus){ + bus.addAll(result.bus) + } + if(result.metro){ + bus.addAll(result.metro) + } + + if(bus){ + row << bus.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.supermarket){ + row << result.supermarket.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.hospital){ + row << result.hospital.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.bank){ + row << result.bank.join(",").replace('"',"") + }else{ + row << "" + } + } + rows << row + } + } + } + ExcelUtils.write(output, rows, "底层数据-小区") + } + + def createPaimai(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"] + def keys = [] + for(def d : datas){ + if(d['Q']) { + def dist = CommonDatautils.findDistrictById(d['Q']) + if (dist) { + def key = dist.city+"#"+dist.region+"#"+dist.name + if(keys.contains(key)){ + continue + } + keys << key + def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10) + if(jsonObject && jsonObject.result){ + def results = jsonObject.result + for(def r: results){ + def row = [] + row << d['A'] + row << d['B'] + row << d['C'] + row << r.matchName + row << r.address + row << r.houseNum + row << r.roomNum + row << r.area + row << r.startDate + row << r.count + row << r.status + row << r.tradeAvgPrice + row << r.tradePrice + row << r.id + rows << row + } + println dist.name +" "+results.size() + } + } + } + } + ExcelUtils.write(output, rows, "拍卖案例") + } + + def createHouse(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "挂牌" + row << r.districtName + if(r.estateType){ + row << r.estateType + }else{ + if(r.purpose){ + row << r.purpose + }else{ + row << "" + } + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.pDate + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + row << r.direction + row << SourceNameUtil.getResourceName(r.root) + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "挂牌案例") + } + + def createEvaluation(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "评估" + row << r.commName + row << "" + + if(r.roomcnt){ + def roomStr = r.roomcnt+"室" + if(r.livingroomcnt){ + roomStr = roomStr +r.livingroomcnt+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.buildArea + row << r.fitment + row << r.cDate.substring(0, r.cDate.indexOf("T")) + row << "" + row << r.unitPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.floorcnt){ + row << r.floorcnt + }else{ + row << "" + } + + if(r.houseaspect && !r.houseaspect.contains("暂")){ + row << r.houseaspect + }else{ + row << "" + } + + row << "房价网" + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "评估案例") + } + + + def createDealPrice(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "成交" + row << r.districtName + if(r.type){ + row << r.type + }else{ + row << "" + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.tradeDate.substring(0, r.tradeDate.indexOf("T")) + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + if(r.direction && !r.direction.contains("暂")){ + row << r.direction + }else{ + row << "" + } + + row << r.source + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "成交案例") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy index 2a2f5890f..63d7f841f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy @@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch { @Override public void process(Object args) { - def cities = ["北京", "上海"] + def cities = ["北京", "上海"]// for (def city : cities) { def ret = CommonDatautils.getAllDistricts("district", city) // def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index acced0959..c7451c5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON import com.fangjia.core.util.DataUtil import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import org.jsoup.Jsoup import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.ChromeDriver @@ -19,9 +22,91 @@ class PaimaiTest { DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { def task = new PaimaiTest(); - task.run(); +// task.run(); +// task.stat() +// task.updatePaimai() + task.updateFieldPaimai() + } + + def updateFieldPaimai(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) + def index = 0 + def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + for(def obj : dbcursor) { + index++ + println index + if(obj.depositPrice){ + obj.put("depositPrice", obj.depositPrice.toInteger()) + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + } + } + } + + def updatePaimai(){ + def q = new BasicDBObject() + def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def index =0 + for(def d : dbcursor){ + index++ + println index + def query = new BasicDBObject() + query.put("url", d.url) + def dbo = paimai43_COLL.findOne(query); + + DBObject obj = new BasicDBObject(); + d.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimai43_COLL.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + } + + def stat(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', ["重庆","重庆市"])) + def dbcursor = paimaiResultColl.find(q) + def keys = [] + for(def db : dbcursor){ + if(db.matchCity){ + def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName; + if(!keys.contains(key)){ + keys << key + } +// try{ +// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// db.matchCity = jsonObject.get("city") +// db.matchRegion = jsonObject.get("district") +// db.matchName = jsonObject.get("name") +// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db)) +// println db.address+" "+db.matchName +// } +// }catch(Exception e){ +// println "error "+db.address +// } + + } + } + println keys.size() } def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy index e5ffbd54f..172cc24b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.loudong +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -16,8 +17,15 @@ class CqfdcChromeCrawl { static main(args) { - def task = new CqfdcChromeCrawl(); - task.run() +// def task = new CqfdcChromeCrawl(); +// task.run() + + def name = "E3号楼" //69幢(规划C-7号楼) 规划5-5号楼 + def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + println s + + def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+单元") + println unitNumStr } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index bd59e65c7..9fbafb2ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -1,10 +1,14 @@ package com.fangjia.script.loudong +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject @@ -14,6 +18,8 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern class CqfdcMatch { @@ -23,6 +29,10 @@ class CqfdcMatch { static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl) + public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { @@ -33,6 +43,7 @@ class CqfdcMatch { // task.matchLoudong() task.addLoudong() +// task.addFdcLoudong() } def export(){ def q = new BasicDBObject() @@ -157,29 +168,378 @@ class CqfdcMatch { return rets; } + def addHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def keys = [] + def index = 0 + for(def d : dbcursor) { + index++ + println index + if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){ + continue + } + if(d.area.toDouble()<=25){ + continue + } + def newRegion = RegionData.formatRegion(d.city, d.region) + if (newRegion) { + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if (houseNumStr) { + house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "") + } + if (house_num) { + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if (unitNum) { + unitNum = unitNum.replace("单元", "") + } + def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num + if(!keys.contains(key) && d.room_num) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num) + if(!count){ + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("house_num_unit", unitNum) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "房谱网") + if(d.floor_heigth && d.floor_heigth.isNumber()){ + udata.put("house_height", d.floor_heigth.toInteger()) + } + if(d.floor && d.floor.isNumber()){ + udata.put("house_floor", d.floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", district_name) + q.put("house_num", house_num) + if(unitNum){ + q.put("unit_num", unitNum) + } + q.put("room_num", room_num) + def count = HOUSE_LOUDONG_LINE_COLL.count(q) + return count; + } + def findAllLoundongByHouseNum(){ def q = new BasicDBObject() - def dbcursor = loudongConn.find(q).batchSize(5000) - def rets = [] - def map = [:] + def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000) + def keys = [] for(def d : dbcursor){ - def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num - if(d.unit_num){ - key = key +"##"+d.unit_num - } - def detailMap= map.get(key) - if(!detailMap){ - detailMap = [:] - map.put(key,detailMap) - } + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if(unitNum){ + unitNum = unitNum.replace("单元","") + } + def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num + if(unitNum){ + key = key +"##"+unitNum + } + if(!keys.contains(key)){ + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num) + println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}" + if(obj){ + def udata = new BasicDBObject() + def areas = obj.areas + if(!areas.contains(d.area)){ + areas << d.area - if(detailMap.height){ + def forms = obj.from + if(!forms.contains("房谱网")){ + forms << "房谱网" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + }else{ + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("unit_num", unitNum) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("areas",[d.area]) + udata.put("from",["房谱网"]) + if(d.floor_heigth && d.floor_heigth.toString().toInteger()){ + udata.put("floors", d.floor_heigth.toString().toInteger()) + } + + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } } } - return rets; } + def addFdcHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000) + def keys = [] + def index = 0 + for(def d : dbcursor){ + index++ + println index + if(!d.use_type){ + continue + } + if(!d.use_type.contains("住宅")){ + continue + } + def newRegion = d.match_region + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num + if(!keys.contains(key)) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num) + if(!count){ + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def floor = null + if(d.room_num && d.room_num.contains("-")) { + def arrs = d.room_num.split("-") + floor = arrs[0] + } + Integer room = null + Integer hall = null + if(d.room_type && d.room_type.contains("室")){ + def roomStr = d.room_type.substring(0,d.room_type.indexOf("室")) + if(roomStr){ + roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + room = roomStr.toInteger() + } + + if(d.room_type.contains("厅")){ + def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅")) + if(hallStr){ + hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + hall = hallStr.toInteger() + } + } + } + + + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if(d.unit_num){ + udata.put("house_num_unit", d.unit_num) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "重庆网上房地产") + udata.put("house_height", maxHeight) + if(room){ + udata.put("room", room) + } + if(hall){ + udata.put("hall", hall) + } + if(floor && floor.isNumber()){ + udata.put("house_floor", floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + public String getMatchString(String text,String pattern){ + Pattern p=Pattern.compile(pattern); + Matcher m=p.matcher(text); + if(m.find()){ + return m.group(0) + } + return null; + } + + def addFdcBaseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + if(!d.match_region){ + continue + } + def newRegion = d.match_region + + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num + if(d.unit_num) { + key = key + "##" + d.unit_num + + if (!keys.contains(key)) { + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num) + println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}" + + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + if (obj) { + + def areas = obj.areas + if (!areas.contains(d.area)) { + areas << d.area + + def forms = obj.from + if (!forms.contains("重庆网上房地产")) { + forms << "重庆网上房地产" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + } else { + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if (d.unit_num) { + udata.put("unit_num", d.unit_num) + } + if (d.address) { + def address = AddressUtil.formatAddress(d.city, d.address) + if (address) { + udata.put("address", address) + } + } + udata.put("areas", [d.area]) + udata.put("from", ["重庆网上房地产"]) + udata.put("floors", maxHeight) + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findMaxHeight(city, name,house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("name",name) + q.put("house_num_desc",house_num) + q.put("unit_num",unit_num) + def dbcursor = loudongFdcConn.find(q) + def heights = [] + for(def h : dbcursor){ + if(h.nominal_floor && h.nominal_floor.isNumber()){ + def height = h.nominal_floor.toInteger() + if(!heights.contains(height)){ + heights << height + } + } + } + if(heights){ + heights = heights.sort() + return heights[heights.size()-1] + } + return null + } + + def findLoudongOnLine(city, region, name, house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + q.put("house_num",house_num) + if(unit_num){ + q.put("unit_num",unit_num) + } + def dbo = LOUDONG_LINE_COLL.findOne(q) + return dbo + } + + def stat(){ def q = new BasicDBObject() @@ -291,7 +651,51 @@ class CqfdcMatch { } def addLoudong(){ +// def stat = statBaseLoudong() +// println stat +// findAllLoundongByHouseNum(); + def stat1 = statHouseLoudong() + println stat1 +// addHouseLoudong() + } + def addFdcLoudong(){ +// addFdcBaseLoudong() + addFdcHouseLoudong() + } + + def statHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.district_id && !ids.contains(d.district_id)){ + ids << d.district_id + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + + def statBaseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){ + ids << d.id_in_seaweed + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 668172385..c17285a78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -128,10 +128,10 @@ class SeaweedUtils { */ static formatHouseholdCount(def household_count) { if (household_count && FieldValidatorUtils.checkValue(household_count) ) { - if (Long.valueOf(household_count) < 50) { + if (Integer.valueOf(household_count) < 50) { return "" // 户数 } - return household_count + return household_count.toInteger() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ee0977a28..c676385ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() - def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", @@ -192,37 +192,37 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] - for(def city : citys){ - def str = cityList.get(city) - for(def i=0;i<1;i++){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } - } - } - -// cityList.each{city, str-> +// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] +// for(def city : citys){ +// def str = cityList.get(city) // for(def i=0;i<1;i++){ -// if(citys.contains(city)){ -// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); -// if(districts){ -// def arrs = str.split("\\|") -// def url = "https://"+arrs[0]+".58.com/xiaoqu/" -// distMap.put(city, districts) -// -// addCrawlKeys(city); -// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); -// } +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); // } // } // } + cityList.each{city, str-> + for(def i=0;i<1;i++){ +// if(citys.contains(city)){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } +// } + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index bd0fbb3f3..4866f25b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202202); - def count2 = findTotalCount(source, 202203); + def count1 = findTotalCount(source, 202203); + def count2 = findTotalCount(source, 202204); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 5ef52da25..29c5f5f4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -229,6 +229,93 @@ public class OpenApiClient { return null } + public static JSONObject searchLoudong(String id, String houseNum, String unitNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("roomNumber", roomNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standard2Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap(); @@ -593,6 +680,90 @@ public class OpenApiClient { } + static JSONObject facilitys(String city, String region, String name) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("region", region) + params.put("name", name) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject searchPaimais(String city, String region, String name, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("district", region) + params.put("name", name) + params.put("limit", "10") + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject searchDealPrice(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject houseList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startTime", startDate) + params.put("pageSize", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject evaluaionList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + static JSONObject districtDetail(String city, String region, String name) { Map params = new HashMap(); params.put("token", TOKEN); From a43af522bb569520754bccc5114141b73e4b9aa5 Mon Sep 17 00:00:00 2001 From: xuhe Date: Thu, 12 May 2022 19:03:28 +0800 Subject: [PATCH 24/83] =?UTF-8?q?=E4=BC=B0=E5=80=BC=E4=BE=9B=E6=95=B0?= =?UTF-8?q?=E6=96=B0=E5=A2=9EMD5=E5=8A=A0=E5=AF=86=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/CcbFtpUtils.groovy | 6 ++++ .../ExportOracleEvaluationData.groovy | 23 +++++++++++++++ .../fangjia/script/gj/ccb/FileUtils.groovy | 28 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 3fffa870c..74474aff6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -57,6 +57,12 @@ class CcbFtpUtils { ftpUpload(dataCodeFileName)//上传加密码文件到FTP } + static def listFileNames(month){ + //上传到ftp + sftpChannel = reConnect(); + Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + return files + } static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index edc128cf1..7258934fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import org.apache.commons.codec.digest.DigestUtils class ExportOracleEvaluationData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -33,6 +35,8 @@ class ExportOracleEvaluationData { exportCommCaseDetailGj(curMonth) def curDate = curMonth.toString()+"28" +// 生成MD5文件 + exportMd5() SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); // ok文件 @@ -413,4 +417,23 @@ class ExportOracleEvaluationData { CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) } + def exportMd5(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = "/ccb/md5.txt" + if(CommonDatautils.isWindow()){ + fileName = "D:\\房价网\\md5.txt" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "md5.txt 上传Ftp完成......" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 3e4981856..23513b80b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -50,4 +50,32 @@ class FileUtils { } out.close(); } + + static def createTxtFile(ret, fileName, charset, isAppend){ + File file = new File(fileName); + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + OutputStream out = new FileOutputStream(fileName, isAppend); + def sb = new StringBuffer(); + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + def index = 0 + for(def row : ret){ + if(row){ + index++ + sb.append(row) + sb.append("\n"); + } + } + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } } From 6a511eff5a70f9318e27551266aa553ff7da35cb Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Thu, 12 May 2022 22:14:28 +0800 Subject: [PATCH 25/83] =?UTF-8?q?=E6=94=BE=E5=BC=80=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=B4=A6=E5=8F=B7=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/processor/CityProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 6f88c20ef..6ae746ab5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -52,8 +52,8 @@ public class CityProcessor extends AbstractAddressProcessor { } if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); - String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null; - String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity; + String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null; + String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity; /**如果省份和城市信息相互矛盾 则用区域推断城市*/ if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){ List possibleCities = speculateCityNameByRegion(address, city); From e1ff6a3ab75b9e1606185d2c6e65452941c3698d Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Thu, 12 May 2022 22:17:19 +0800 Subject: [PATCH 26/83] =?UTF-8?q?=E6=94=BE=E5=BC=80=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=B4=A6=E5=8F=B7=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/util/DbUtils.groovy | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 3b83880cd..c39e16b2e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -92,11 +92,6 @@ public class DbUtils{ } } - - public static void executeQueryOnSeawater(BasicDBObject query,Closure c, BasicDBObject fields = null){ - seawater.find(query,fields).batchSize(10000).each(c) - } - public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){ SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c) } @@ -171,10 +166,6 @@ public class DbUtils{ return SEAWEED_SLAVE.findOne(query,fields); } - public static int executeCountOnSeawater(BasicDBObject query){ - return seawater.count(query); - } - public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){ BasicDBObject query = new BasicDBObject() if(city){ @@ -350,31 +341,6 @@ public class DbUtils{ return district.lat && district.lng && district.lng > 0 && district.lat > 0 } -/** - * 记录修改小区库的操作日志 - * @param city - * @param region - * @param name - * @return - */ - def static saveDeleteLog(String city, String region,String name,String category,int status){ - if(status!=1){ - return - } - DBObject obj = new BasicDBObject() - obj.put("city", city); - obj.put("region", region); - obj.put("district_name", name); - obj.put("category" , category) - obj.put("origin" , "脚本") - if(status==1){ - obj.put("type" , "remove") - } - obj.put("c_date" , new Date()) - obj.put("m_date" , new Date()) - OPERATION_LOG.save(obj) - } - public static DBCollection getCollection(String host, int port, String dbName ,String collName){ def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"] if(!collection){ From 0fab2bc076264571d8e4dd2db53db1b076fef94b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 12 May 2022 23:24:31 +0800 Subject: [PATCH 27/83] =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/core/util/SeaweedUtils.java | 202 ++++++++++++++++++ .../dal/mongo/fangjia/model/PaiMai.java | 27 +++ .../open/controllers/AuctionController.java | 39 ++++ .../open/controllers/PropertyController.java | 10 +- 4 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java new file mode 100644 index 000000000..698833bd5 --- /dev/null +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java @@ -0,0 +1,202 @@ +package com.fangjia.core.util; + +import org.apache.commons.lang.StringUtils; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SeaweedUtils { + public static void main(String[] args){ + System.out.println(formatVirescencePercent("1")); + } + /** + * 标准化输出绿化率 + */ + public static String formatVirescencePercent(String virescence_percent) { + // 绿化率 + if (StringUtils.isNotBlank(virescence_percent) && checkValue(virescence_percent)!=null) { + double val = Double.valueOf(virescence_percent); + if (val < 1) { + return String.valueOf(val * 100); + } + if (val == 1) { + return String.valueOf(val * 10); + } + if (val >= 70) { + return "65"; + } + return String.valueOf(val); + } + return null; + } + + /** + * 标准化容积率输出 + */ + public static String formatCubagePercent(String cubage_percent) { + if (StringUtils.isNotBlank(cubage_percent) && checkValue(cubage_percent)!=null) { + double val = Double.valueOf(cubage_percent); + if (val >= 7 && val < 8) { + return "6.2"; + } + if (val >= 8 && val < 9) { + return "6.3"; + } + if (val >= 9 && val < 10) { + return "6.5"; + } + if (val >= 10) { + return "6.7"; + } + return String.valueOf(val); + } + return null; + } + + /** + * 标准化车位输出 + */ + public static String formatCarSiteCount(String car_site_count) { + if (StringUtils.isNotBlank(car_site_count) && checkValue(car_site_count)!=null) { + long val = Long.valueOf(car_site_count); + if (val > 2500 || val < 100) { + return ""; + } + return String.valueOf(val); + } + return null; + } + + /** + * 标准化户数输出 + */ + public static String formatHouseholdCount(String household_count) { + if (StringUtils.isNotBlank(household_count) && checkValue(household_count)!=null ) { + if (Integer.valueOf(household_count) < 50) { + return null; // 户数 + } + return household_count; + } + return null; + } + + /** + * 标准化总占地面积输出 + */ + public static String formatAllArea(String all_area) { + if (StringUtils.isNotBlank(all_area)) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + String val = all_area.toString(); + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000); + } else if(checkValue(all_area)!=null) { + formattedVal = Double.valueOf(all_area); + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return null; + } + return String.valueOf(formattedVal); + } + return null; + } + + /** + * 提取浮点数与整数 + */ + static String getNumber(String input) { + Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?"); + Matcher matcher = pattern.matcher(input); + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>0){ + return list.get(0); + } + return null; + } + + + + + static Float checkValue(String value){ + if(StringUtils.isNotBlank(value)){ + value = getNumber(value); + if(StringUtils.isNotBlank(value)){ + value = subZeroAndDot(value); + if(isNumber(value)){ + Float v = Float.parseFloat(value); + if(v>0){ + return v; + }else{ + return null; + } + }else if(isFloat(value)){ + Float v = Float.parseFloat(value); + if(v>0){ + return v; + }else{ + return null; + } + } else{ + return null; + } + } + } + return null; + } + + static boolean isNumber(String str){ + try { + Integer.parseInt(str); + } catch (NumberFormatException e) { + return false; + } + return true; + } + + static boolean isFloat(String str){ + try { + Float.parseFloat(str); + } catch (NumberFormatException e) { + return false; + } + return true; + } + + /** + * 正则表达式去掉多余的.与0 + * @param + * @return + */ + static String subZeroAndDot(String v){ + + if(StringUtils.isNotBlank(v)){ + if(v.indexOf(".")!=-1){ + Double myNum2 = Double.valueOf(v); + BigDecimal b = new java.math.BigDecimal(myNum2); + Double myNum3 = b.setScale(2, java.math.BigDecimal.ROUND_HALF_UP).doubleValue(); + v = myNum3.toString(); + } + + if(v.indexOf(".")>-1){ + while(true) + { + if(v.endsWith("0")){ + v = v.substring(0,v.length()-1); + }else{ + break; + } + } + if(v.endsWith(".")){ + v=v.substring(0,v.length()-1); + } + } + } + return v; + } + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java index fa5f7cb2f..e7a406dc8 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java @@ -99,6 +99,9 @@ public class PaiMai implements MongoEntity { private Integer marketAvgPrice; /**物业类型*/ private String matchEstateType; + private String houseNum; + private String unitNum; + private String roomNum; @Id @GeneratedValue @@ -488,4 +491,28 @@ public class PaiMai implements MongoEntity { public void setMatchEstateType(String matchEstateType) { this.matchEstateType = matchEstateType; } + + public String getHouseNum() { + return houseNum; + } + + public void setHouseNum(String houseNum) { + this.houseNum = houseNum; + } + + public String getUnitNum() { + return unitNum; + } + + public void setUnitNum(String unitNum) { + this.unitNum = unitNum; + } + + public String getRoomNum() { + return roomNum; + } + + public void setRoomNum(String roomNum) { + this.roomNum = roomNum; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index ed4a73f2d..8d09d50b1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -1,6 +1,7 @@ package com.fangjia.open.controllers; import com.fangjia.core.util.DateUtil; +import com.fangjia.core.util.HttpRequest; import com.fangjia.dal.QueryParam; import com.fangjia.dal.SortParam; import com.fangjia.dal.common.DistrictKey; @@ -14,6 +15,7 @@ import com.fangjia.open.model.BasicModel; import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.fsh.DistrictNewPriceService; import com.fangjia.service.open.PaiMaiService; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.bson.types.ObjectId; @@ -204,10 +206,47 @@ public class AuctionController { List sortParams = new ArrayList(); sortParams.add(new SortParam("startDate", false)); List paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS); + if(paimais!=null && paimais.size()>0){ + for(PaiMai paiMai : paimais){ + Map map = standard(paiMai.getCity(), paiMai.getAddress()); + if(map!=null){ + paiMai.setHouseNum((String) map.get("buildingNumber")); + paiMai.setUnitNum((String) map.get("unitNumber")); + paiMai.setRoomNum((String) map.get("roomNumber")); + } + } + } mav.setModel(BasicModel.buildOKModel().setResult(paimais)); return mav; } + + private Map standard(String city, String address){ + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + if(json.containsKey("result")){ + return (Map)json.get("result"); + } + } + return null; + } + public void setPaiMaiService(PaiMaiService paiMaiService) { this.paiMaiService = paiMaiService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 934ca4cdb..0b0dce39f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -734,13 +734,13 @@ public class PropertyController { districtInfoDto.setCommunity(district.getStreetCommunity()); districtInfoDto.setCommercialArea(district.getBusinessQuarter()); if (district.getHouseholdCount() != null) { - districtInfoDto.setTotalUnitCount(district.getHouseholdCount().toString()); + districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString())); } else { districtInfoDto.setTotalUnitCount(""); } districtInfoDto.setManagementFee(district.getEstateCharge()); if (district.getCarSiteCount() != null) { - districtInfoDto.setParking(district.getCarSiteCount().toString()); + districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString())); } else { districtInfoDto.setParking(""); } @@ -761,10 +761,10 @@ public class PropertyController { districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate()); districtInfoDto.setMoveinTime(district.getLiveDate()); - districtInfoDto.setFloorSpace(district.getAllArea()); + districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea())); districtInfoDto.setConstructionSpace(district.getBuildArea()); - districtInfoDto.setPlotRatio(district.getCubagePercent()); - districtInfoDto.setGreenPercentage(district.getVirescencePercent()); + districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent())); + districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent())); districtInfoDto.setPreSalePermit(district.getLicence()); districtInfoDto.setDeveloper(district.getDeveloper()); districtInfoDto.setDiscount(district.getDiscount()); From 606c17bbe76bdd8a53fa4cf2fee38685b64b45fe Mon Sep 17 00:00:00 2001 From: xuhe Date: Sat, 14 May 2022 09:20:01 +0800 Subject: [PATCH 28/83] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E8=9E=8D=E5=90=88?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BB=BA=E7=AD=91=E7=BB=93=E6=9E=84=E5=AD=97?= =?UTF-8?q?=E5=85=B8=E5=80=BC=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/ccb/CcbCodeUtils.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 1dabf3ada..66ff6a8f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -199,18 +199,18 @@ class CcbCodeUtils { static def getBuildStructureCode(def buildStructure){ if(buildStructure){ if(buildStructure.contains("钢结构")){ - return "001"; + return "3"; }else if(buildStructure.contains("钢、钢筋混凝土结构")){ - return "002"; + return "1"; }else if(buildStructure.contains("钢筋混凝土结构")){ - return "003"; + return "1"; }else if(buildStructure.contains("砖木结构")){ - return "004"; + return "10"; }else if(buildStructure.contains("混合结构")){ - return "005"; + return "2"; } } - return "006"; + return "11"; } static def getDistHouseTypeCode(def estateType){ From 6af5da836c28eecdea6886146398c42660006628 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Wed, 18 May 2022 15:45:01 +0800 Subject: [PATCH 29/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/AddressParserServiceImpl.java | 28 +++--- .../fangjia/service/address/mode/Address.java | 18 ++++ .../address/processor/AddressProcessors.java | 4 +- .../address/processor/CityProcessor.java | 22 ++--- .../address/processor/DistrictProcessor.java | 3 + .../address/processor/GeoProcessor.java | 87 ++++++++----------- .../processor/VillageNameProcessor.java | 9 -- .../service/cnAnalyzer/CnTokenizer.java | 24 ++--- 8 files changed, 93 insertions(+), 102 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 46545fc04..1d5947440 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -759,6 +759,13 @@ public class AddressParserServiceImpl implements AddressParserService { double simScore; boolean onSameRoad = false; boolean sameDistrict = isSameDistrict(address, metaData); + if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){ + return 0; + } + //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区 + if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){ + return 0; + } Iterator> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator(); Map.Entry addrEntry; while(addrEntrySet.hasNext()){ @@ -808,7 +815,6 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - String keywordPinYin = address.getKeywordsPinYin(); String keywords = address.getKeywords(); String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin(); @@ -904,23 +910,25 @@ public class AddressParserServiceImpl implements AddressParserService { double matchLevelScore = 0; double similarityScore = 0; - if (standardAddressScore > 0 && keywordScore > 0) { + if(metaData.getId().equals(address.getPolygonSeaweedId())){ matchLevelScore = 0.9; - similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; - } else if(keywordScore > 0 && streetScore > 0){ + similarityScore = 0.05; + }else if (standardAddressScore > 0 && keywordScore > 0) { matchLevelScore = 0.8; - similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; + similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; } else if (standardAddressScore > 0) { matchLevelScore = 0.7; similarityScore = standardAddressScore * SIMILARITY_RATE; + } else if(keywordScore > 0 && streetScore > 0){ + matchLevelScore = 0.6; + similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (keywordScore > 0) { matchLevelScore = 0.6; similarityScore = keywordScore * SIMILARITY_RATE; } else if (fuzzyKeywordsScore > 0) { - matchLevelScore = 0.5; + matchLevelScore = 0.4; similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; } - if (similarityScore > 0) { if(metaData.getBuildingType() != null){ if ("shop".equals(metaData.getBuildingType().toString())) { @@ -942,10 +950,6 @@ public class AddressParserServiceImpl implements AddressParserService { * similarityScore = 相似度部分(0.07) + 类型部分(0.01) < 0.1 * */ double score = matchLevelScore + similarityScore; - if(address.hasCoordinate() && metaData.hasCoordinate() - && GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){ - score = 0d; - } return score; } @@ -980,7 +984,7 @@ public class AddressParserServiceImpl implements AddressParserService { return false; } if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){ - return false; + return true; } String district = metaData.getDistrict(); return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 720da51bb..ed723c1d5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -61,6 +61,8 @@ public class Address { private String suggestDistrict; private String suggestName; private String suggestNamePinYin; + private String polygonSeaweedId; + private String geoRegion; private List buildingNumbers; private PreciseLevel preciseLevel; private boolean pinYin; @@ -750,6 +752,14 @@ public class Address { return false; } + public String getGeoRegion() { + return geoRegion; + } + + public void setGeoRegion(String geoRegion) { + this.geoRegion = geoRegion; + } + @Override public String toString() { return "Address{" + @@ -809,6 +819,14 @@ public class Address { '}'; } + public String getPolygonSeaweedId() { + return polygonSeaweedId; + } + + public void setPolygonSeaweedId(String polygonSeaweedId) { + this.polygonSeaweedId = polygonSeaweedId; + } + public String toMd5Hex(){ String keyProperties = "Address{" + "city='" + city + '\'' + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java index aa72e6d01..9097a3a63 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java @@ -20,11 +20,9 @@ public class AddressProcessors{ static { //修改顺序需要调整process i 的值 - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new ProximalCharactersProcessor()); addressProcessors.add(new PreNoiseProcessor()); addressProcessors.add(new AddressFormatProcessor()); - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new FloorProcessor()); addressProcessors.add(new TooltipProcessor()); addressProcessors.add(new ProvinceProcessor()); @@ -44,7 +42,7 @@ public class AddressProcessors{ addressProcessors.add(new NumericProcessor()); addressProcessors.add(new ProvinceAdjustProcessor()); addressProcessors.add(new FormatProcessor()); -// addressProcessors.add(new GeoProcessor()); + addressProcessors.add(new GeoProcessor()); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 6ae746ab5..130b7eea9 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -52,6 +52,7 @@ public class CityProcessor extends AbstractAddressProcessor { } if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); + address.setCity(formatCity); String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null; String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity; /**如果省份和城市信息相互矛盾 则用区域推断城市*/ @@ -191,7 +192,6 @@ public class CityProcessor extends AbstractAddressProcessor { }else{ weight.score += 2; } - }else if(address.contains(city)){ index = address.indexOf(city); weight.index = index; @@ -208,9 +208,11 @@ public class CityProcessor extends AbstractAddressProcessor { if(matchResult !=null){ address = address.replace(matchResult.matchContent, ""); } - if(address.indexOf(region, index) > -1){ + if(address.indexOf(region + "区", index) > -1 || address.indexOf(region + "县", index) > -1){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ weight.score += 2; + }else if(city.equals(addr.getCity())){ + weight.score += 10; }else{ weight.score += 1; } @@ -223,7 +225,11 @@ public class CityProcessor extends AbstractAddressProcessor { for(String alia : alias){ if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1 && (alia.endsWith("市")|(alia.endsWith("县")))){ - weight.score += 1; + if(city.equals(addr.getCity())){ + weight.score += 10; + }else{ + weight.score += 1; + } weight.index = address.indexOf(alia, index); break; } @@ -237,16 +243,6 @@ public class CityProcessor extends AbstractAddressProcessor { } List weights = new ArrayList(weightMap.values()); if(weights.size() > 0){ - /*String street = getStreet(address); - if(street != null){ - Iterator iterator = weights.iterator(); - while(iterator.hasNext()){ - FragmentWeight fragmentWeight = iterator.next(); - if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){ - iterator.remove(); - } - } - }*/ Collections.sort(weights); } if(weights.size() > 0){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java index 5cfba96b5..697e69f8d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java @@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor { address.setDevelopmentZone(dz); } } + if(address.getSuggestDistrict() != null && address.getDistrict() == null){ + address.setDistrict(address.getSuggestDistrict()); + } address.setAddress(orgAddress); address.setEffectiveAddress(address.getAddress()); return RepeatStatus.NEXT; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index 83d74a0d4..c72141550 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -5,8 +5,11 @@ import com.fangjia.core.util.HttpRequest; import com.fangjia.service.address.mode.Address; import net.sf.json.JSONArray; import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import java.util.*; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * 地址经纬度处理器 @@ -18,7 +21,6 @@ public class GeoProcessor extends AbstractAddressProcessor { private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); - List DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); @Override public RepeatStatus process(Address address) { @@ -26,68 +28,55 @@ public class GeoProcessor extends AbstractAddressProcessor { if (address.hasCoordinate()) { return RepeatStatus.NEXT; } - List queryTypes = new ArrayList(); - List addressList = new ArrayList(); - if (address.isStandardAddress()) { - addressList.add(address.getStandardAddress()); - queryTypes.add("StandardAddress"); - } - if (StringUtils.isNotBlank(address.getOrgKeywords())) { - addressList.add(address.getOrgKeywords()); - queryTypes.add("Keywords"); - } HttpRequest request = new HttpRequest(); Map params = new HashMap(); - params.put("address", StringUtils.join(addressList, "|")); + params.put("address", address.getOrgAddress()); params.put("key", KEY); params.put("city", address.getCity()); - params.put("batch", "true"); params.put("output", "json"); try { JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8"); if (response.getInt("status") == 1) { JSONArray geocodes = response.getJSONArray("geocodes"); - if(queryTypes.contains("StandardAddress")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); + if(geocodes.size() > 0){ + JSONObject json = geocodes.getJSONObject(0); + if(json.get("district") instanceof String){ + String city = address.getCity(); + if(city == null){ + city = RegionData.formatCity(json.getString("city")); } + String region = RegionData.formatRegion(city, json.getString("district")); + address.setGeoRegion(region); } - if(geocode.containsKey("street") && geocode.containsKey("number")){ - String geocodeAddress = geocode.getString("street") + geocode.getString("number"); - if(address.getStandardAddress().contains(geocodeAddress)){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); - } - } - } - if(queryTypes.contains("Keywords")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); - } - } - if(POI_LEVELS.contains(level)){ - String formattedAddress = geocode.getString("formatted_address"); - formattedAddress = formattedAddress - .replaceFirst(geocode.getString("province"), "") - .replaceFirst(geocode.getString("city"), "") - .replaceFirst(geocode.getString("district"), ""); - if(formattedAddress.equals(address.getOrgKeywords())){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); + if(POI_LEVELS.contains(json.getString("level"))){ + //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼 + if("门牌号".equals(json.getString("level"))){ + if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ + return RepeatStatus.NEXT; + } } + String location = json.getString("location"); + address.setLng(Double.valueOf(location.split(",")[0])); + address.setLat(Double.valueOf(location.split(",")[1])); + String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}"; + tmp = tmp.replace("location", location); + Map headers = new HashMap(); + headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz"); + JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8")); + if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){ + JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits"); + if(hits.size() > 0){ + String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id"); + address.setPolygonSeaweedId(seaweedId); + } + } + } } + } } catch (Exception e) { - + e.printStackTrace(); } return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java index 5236b7d55..797d9e367 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java @@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor { @Override public AbstractAddressProcessor.RepeatStatus process(Address address) { String orgAddress = address.getAddress(); - if(address.getTokens()!=null){ - for(Token token:address.getTokens()){ - if("district".equals(token.getKeywordTypes())){ - address.setKeywords(token.getName()); - address.setAddress(orgAddress.replace(token.getName(),"")); - return RepeatStatus.NEXT; - } - } - } List names = tokenDistrictNames(orgAddress); List kws = new ArrayList(); for(String name : names){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java index de010e201..c216a1eba 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java @@ -1,24 +1,16 @@ package com.fangjia.service.cnAnalyzer; -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fangjia.core.util.DataUtil; import com.goojia.solr.ganalytics.IKSegmentation; import com.goojia.solr.ganalytics.Lexeme; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.util.*; /** * 房价网分词器 From 4f9f04418249e8a5ca266c3d556bfd7034303dda Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Wed, 18 May 2022 17:58:36 +0800 Subject: [PATCH 30/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/address/AddressParserServiceImpl.java | 8 ++++---- .../service/address/processor/GeoProcessor.java | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 1d5947440..bd0be4380 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -762,10 +762,10 @@ public class AddressParserServiceImpl implements AddressParserService { if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){ return 0; } - //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区 - if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){ - return 0; - } +// //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区 +// if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){ +// return 0; +// } Iterator> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator(); Map.Entry addrEntry; while(addrEntrySet.hasNext()){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index c72141550..d0c3e88ac 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -20,7 +20,7 @@ public class GeoProcessor extends AbstractAddressProcessor { private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; - List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); + List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号"); @Override public RepeatStatus process(Address address) { @@ -50,11 +50,11 @@ public class GeoProcessor extends AbstractAddressProcessor { } if(POI_LEVELS.contains(json.getString("level"))){ //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼 - if("门牌号".equals(json.getString("level"))){ - if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ - return RepeatStatus.NEXT; - } - } +// if("门牌号".equals(json.getString("level"))){ +// if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ +// return RepeatStatus.NEXT; +// } +// } String location = json.getString("location"); address.setLng(Double.valueOf(location.split(",")[0])); address.setLat(Double.valueOf(location.split(",")[1])); From 83c0f3bfffc5db0fcfe7b8f8f71a2487eb3792ff Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 18 May 2022 18:35:40 +0800 Subject: [PATCH 31/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../OracleDistrict2Mongo85.groovy | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy new file mode 100644 index 000000000..b143277b9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.lucene.queryparser.classic.QueryParser +import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * oralce中小区统计数据同步至85中 + */ +class OracleDistrict2Mongo85 { + + static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017) + + static main(args) { + // 只写入重庆市数据 + def sql = "select distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ WHERE CITYID='500000'" + def count = 0 + sqlInstance.eachRow(sql, { r -> + def region = RegionData.formatRegion("重庆市", r['distname']) + DBObject dbObject = new BasicDBObject("_id", r['commid']) + DBObject updateObject = new BasicDBObject("city", "重庆").append("region", region) + .append("name", r['commname']) + .append('subway', Integer.parseInt(r['subway'].toString())) + .append("traffic", Integer.parseInt(r['traffic'].toString())) + .append("bank", Integer.parseInt(r['bank'].toString())) + .append("primary", Integer.parseInt(r['primary'].toString())) + .append("junior", Integer.parseInt(r['junior'].toString())) + .append("hospital", Integer.parseInt(r['hospital'].toString())) + .append("funeral", Integer.parseInt(r['funeral'].toString())) + .append("c_date", new Date()); + communityPoiConn.update(dbObject, updateObject, true, false) + count++; + if (count % 500 == 0) { + println "已写入" + count + "条小区数据" + } + }); + } +} From 6f69ca6a3394b0f2ec2d4228436a0d615515a966 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 18 May 2022 19:55:28 +0800 Subject: [PATCH 32/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/mongo/open/dao/CommunityPoiDao.java | 9 ++ .../open/dao/impl/CommunityPoiDaoImpl.java | 10 ++ .../dal/mongo/open/model/CommunityPoi.java | 135 ++++++++++++++++++ .../open/controllers/PropertyController.java | 23 +++ .../service/open/CommunityPoiService.java | 8 ++ .../open/impl/CommunityPoiServiceImpl.java | 19 +++ 6 files changed, 204 insertions(+) create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java new file mode 100644 index 000000000..97f9d48de --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.CountDistrictStatBycity; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityPoiDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java new file mode 100644 index 000000000..e8a317b04 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.dal.mongo.tool.model.ApiVisitCount; + +public class CommunityPoiDaoImpl extends MongoDaoImpl implements CommunityPoiDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java new file mode 100644 index 000000000..8427b5aa7 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java @@ -0,0 +1,135 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; +import java.util.Date; + +/** + * 小区周边配套统计信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "fangjia_stat.community_poi_gj") +public class CommunityPoi implements MongoEntity { + + private static final long serialVersionUID = -7663644539542318409L; + + private String id; + private String city; + private String region; + private String name; + private Integer subway; + private Integer traffic; + private Integer bank; + private Integer primary; + private Integer junior; + private Integer hospital; + /** + * 不利设施,丧葬 + */ + private Integer funeral; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "subway") + public Integer getSubway() { + return subway; + } + + public void setSubway(Integer subway) { + this.subway = subway; + } + + @Column(name = "traffic") + public Integer getTraffic() { + return traffic; + } + + public void setTraffic(Integer traffic) { + this.traffic = traffic; + } + + @Column(name = "bank") + public Integer getBank() { + return bank; + } + + public void setBank(Integer bank) { + this.bank = bank; + } + + @Column(name = "primary") + public Integer getPrimary() { + return primary; + } + + public void setPrimary(Integer primary) { + this.primary = primary; + } + + @Column(name = "junior") + public Integer getJunior() { + return junior; + } + + public void setJunior(Integer junior) { + this.junior = junior; + } + + @Column(name = "hospital") + public Integer getHospital() { + return hospital; + } + + public void setHospital(Integer hospital) { + this.hospital = hospital; + } + + @Column(name = "funeral") + public Integer getFuneral() { + return funeral; + } + + public void setFuneral(Integer funeral) { + this.funeral = funeral; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index f6e9855f0..b62c50cd5 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -31,12 +31,14 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; import com.fangjia.dal.mongo.model.*; +import com.fangjia.dal.mongo.open.model.CommunityPoi; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; +import com.fangjia.service.open.impl.CommunityPoiServiceImpl; import com.fangjia.service.util.DistrictScoreUtil; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -175,6 +177,7 @@ public class PropertyController { private SeawaterDao seawaterDao; private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; + private CommunityPoiServiceImpl communityPoiService; private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; @@ -1266,6 +1269,18 @@ public class PropertyController { modelFeaturesFto.put("buildYear", String.valueOf(buildYear)); } modelFeaturesFto.append("nearCommunity", nearCommunityDtos); + + // 模型特征新增周边数量 + CommunityPoi communityPoi = communityPoiService.getById(district.getId()); + if (communityPoi != null) { + modelFeaturesFto.put("subway", communityPoi.getSubway()); + modelFeaturesFto.put("traffic", communityPoi.getTraffic()); + modelFeaturesFto.put("bank", communityPoi.getBank()); + modelFeaturesFto.put("primary", communityPoi.getPrimary()); + modelFeaturesFto.put("hospital", communityPoi.getHospital()); + modelFeaturesFto.put("junior", communityPoi.getJunior()); + modelFeaturesFto.put("funeral", communityPoi.getFuneral()); + } districtPriceMap.put("modelFeatures", modelFeaturesFto); } mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); @@ -5601,4 +5616,12 @@ public class PropertyController { public void setRentDealPriceService(RentDealPriceService rentDealPriceService) { this.rentDealPriceService = rentDealPriceService; } + + public CommunityPoiServiceImpl getCommunityPoiService() { + return communityPoiService; + } + + public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) { + this.communityPoiService = communityPoiService; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java new file mode 100644 index 000000000..c0eb458f1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java @@ -0,0 +1,8 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityPoiService { + + CommunityPoi getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java new file mode 100644 index 000000000..8ea0b05e1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java @@ -0,0 +1,19 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityPoiServiceImpl implements CommunityPoiService { + + private CommunityPoiDao communityPoiDao; + + @Override + public CommunityPoi getById(String id) { + return communityPoiDao.findById(id); + } + + public void setCommunityPoiDao(CommunityPoiDao communityPoiDao) { + this.communityPoiDao = communityPoiDao; + } +} From b68dca74dbb778e45c66298901b7079f838d416d Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 18 May 2022 20:19:14 +0800 Subject: [PATCH 33/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A4=9A=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/OracleDistrict2Mongo85.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy index b143277b9..0855b1a8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData +import com.fangjia.core.util.DistrictCodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils @@ -40,12 +41,13 @@ class OracleDistrict2Mongo85 { static main(args) { // 只写入重庆市数据 - def sql = "select distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ WHERE CITYID='500000'" + def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ" def count = 0 sqlInstance.eachRow(sql, { r -> - def region = RegionData.formatRegion("重庆市", r['distname']) + def city = DistrictCodeUtils.getCityName(r['cityid']) + def region = RegionData.formatRegion(city, r['distname']) DBObject dbObject = new BasicDBObject("_id", r['commid']) - DBObject updateObject = new BasicDBObject("city", "重庆").append("region", region) + DBObject updateObject = new BasicDBObject("city", city).append("region", region) .append("name", r['commname']) .append('subway', Integer.parseInt(r['subway'].toString())) .append("traffic", Integer.parseInt(r['traffic'].toString())) From 40e923678a7c69dbf8fbec913bc30e5844dea2b8 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Wed, 18 May 2022 21:21:40 +0800 Subject: [PATCH 34/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/processor/GeoProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index d0c3e88ac..4333ad444 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -5,6 +5,7 @@ import com.fangjia.core.util.HttpRequest; import com.fangjia.service.address.mode.Address; import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.apache.log4j.Logger; import java.util.Arrays; import java.util.HashMap; @@ -19,7 +20,7 @@ import java.util.Map; public class GeoProcessor extends AbstractAddressProcessor { private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; - + private static final Logger logger = Logger.getLogger(GeoProcessor.class); List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号"); @Override @@ -36,6 +37,7 @@ public class GeoProcessor extends AbstractAddressProcessor { params.put("output", "json"); try { JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8"); + logger.info("高德API:" + response.toString()); if (response.getInt("status") == 1) { JSONArray geocodes = response.getJSONArray("geocodes"); if(geocodes.size() > 0){ From 0c4b469ea5c224f20108362e1844783ce6d8a037 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Wed, 18 May 2022 22:04:17 +0800 Subject: [PATCH 35/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/processor/GeoProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index 4333ad444..0ffdbde20 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -25,7 +25,7 @@ public class GeoProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { - + logger.info("Geo Start"); if (address.hasCoordinate()) { return RepeatStatus.NEXT; } @@ -78,6 +78,7 @@ public class GeoProcessor extends AbstractAddressProcessor { } } catch (Exception e) { + logger.error(e); e.printStackTrace(); } return RepeatStatus.NEXT; From b4d20ca24bdcd3225d42a65715ea8c4ce9c7a598 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Thu, 19 May 2022 08:47:41 +0800 Subject: [PATCH 36/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/AddressParserServiceImpl.java | 21 ++++++++++++------- .../address/processor/GeoProcessor.java | 17 ++++++++------- .../address/processor/TooltipProcessor.java | 6 ++---- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index bd0be4380..6d9568d3d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -898,7 +898,7 @@ public class AddressParserServiceImpl implements AddressParserService { if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){ keywordScore = Math.max(keywordScore, namePriority); }else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){ - double similar = AddressParserUtil.getSimilarityRatio(name, address.getEffectiveAddress()); + double similar = AddressParserUtil.getSimilarityRatio(name, address.getOrgAddress()); if(similar == 1){ keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001; }else if(similar > 0.1){ @@ -910,23 +910,28 @@ public class AddressParserServiceImpl implements AddressParserService { double matchLevelScore = 0; double similarityScore = 0; - if(metaData.getId().equals(address.getPolygonSeaweedId())){ + if (standardAddressScore > 0 && keywordScore > 0) { matchLevelScore = 0.9; - similarityScore = 0.05; - }else if (standardAddressScore > 0 && keywordScore > 0) { - matchLevelScore = 0.8; similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; } else if (standardAddressScore > 0) { - matchLevelScore = 0.7; + matchLevelScore = 0.8; similarityScore = standardAddressScore * SIMILARITY_RATE; + }else if(metaData.getId().equals(address.getPolygonSeaweedId())){ + matchLevelScore = 0.7; + similarityScore = 0.05; } else if(keywordScore > 0 && streetScore > 0){ matchLevelScore = 0.6; similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (keywordScore > 0) { - matchLevelScore = 0.6; + matchLevelScore = 0.5; similarityScore = keywordScore * SIMILARITY_RATE; } else if (fuzzyKeywordsScore > 0) { - matchLevelScore = 0.4; + if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(), + metaData.getLng(), metaData.getLat()) < 100){ + matchLevelScore = 0.6; + }else{ + matchLevelScore = 0.4; + } similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; } if (similarityScore > 0) { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index 0ffdbde20..e3ef9a3da 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -25,13 +25,16 @@ public class GeoProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { - logger.info("Geo Start"); if (address.hasCoordinate()) { return RepeatStatus.NEXT; } + String addr = address.getEffectiveAddress(); + if(addr == null || addr.length() < 3){ + addr = address.getOrgAddress(); + } HttpRequest request = new HttpRequest(); Map params = new HashMap(); - params.put("address", address.getOrgAddress()); + params.put("address", addr); params.put("key", KEY); params.put("city", address.getCity()); params.put("output", "json"); @@ -52,11 +55,11 @@ public class GeoProcessor extends AbstractAddressProcessor { } if(POI_LEVELS.contains(json.getString("level"))){ //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼 -// if("门牌号".equals(json.getString("level"))){ -// if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ -// return RepeatStatus.NEXT; -// } -// } + if("门牌号".equals(json.getString("level"))){ + if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ + return RepeatStatus.NEXT; + } + } String location = json.getString("location"); address.setLng(Double.valueOf(location.split(",")[0])); address.setLat(Double.valueOf(location.split(",")[1])); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java index 4d3b65b14..3921fddf5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java @@ -42,12 +42,10 @@ public class TooltipProcessor extends AbstractAddressProcessor { if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){ address.setTooltip(tooltip); address.setStreet(getStreet(tooltip)); - } else if(containsDistrictName(tooltip)){ - address.setTooltip(tooltip); - } else if(RegionData.isRegion(address.getCity(), tooltip)){ + }else if(RegionData.isRegion(address.getCity(), tooltip)){ address.setSuggestDistrict(tooltip); }else{ - address.addNosie(tooltip); + address.setTooltip(tooltip); } } } From 9bfe8cdb5fa24f8529e66d3bf3ca9ec5eec0145e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 19 May 2022 10:14:12 +0800 Subject: [PATCH 37/83] update --- .../dal/seawater/dao/SeawaterDaoImpl.java | 18 +++++++++++++----- .../fangjia/dal/seawater/model/Seawater.java | 10 ++++++++++ .../open/controllers/AuctionController.java | 9 +++++++++ .../open/controllers/HouseController.java | 4 +++- .../com/fangjia/open/model/AuctionModel.java | 9 +++++++++ 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 203b6818b..bf43fa450 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -238,6 +238,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(source.containsKey("root")){ seawater.setRoot(source.getString("root")); } + + if(source.containsKey("estate_type")){ + seawater.setEstateType(source.getString("estate_type")); + } + + if(source.containsKey("build_type")){ + seawater.setBuildType(source.getString("build_type")); + } seawaterList.add(seawater); } return seawaterList; @@ -386,11 +394,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } // 不展示删除 - JSONObject param = new JSONObject(); - param.put("status", 0); - JSONObject temp = new JSONObject(); - temp.put("term", param); - jsonList.add(temp); +// JSONObject param = new JSONObject(); +// param.put("status", 0); +// JSONObject temp = new JSONObject(); +// temp.put("term", param); +// jsonList.add(temp); query.put("must", jsonList); q.put("bool", query); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index 48005c87b..910d805ea 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -60,6 +60,7 @@ public class Seawater implements SeawaterEntity { private Integer avgPrice; private String purpose; private String estateType; + private String buildType; @Id @Column(name = "_id") @@ -305,4 +306,13 @@ public class Seawater implements SeawaterEntity { public void setEstateType(String estateType) { this.estateType = estateType; } + + @Column(name = "build_type") + public String getBuildType() { + return buildType; + } + + public void setBuildType(String buildType) { + this.buildType = buildType; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index 8d09d50b1..f2a10c491 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -203,6 +203,15 @@ public class AuctionController { if(StringUtils.isNotBlank(model.getName())){ params.add(new QueryParam("matchName", model.getName())); } + + Date startDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + if(startDate != null){ + params.add(new QueryParam("startDate", startDate)); + } + } + List sortParams = new ArrayList(); sortParams.add(new SortParam("startDate", false)); List paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index bdf38d8b0..50b2700a2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -421,7 +421,9 @@ public class HouseController { district = districtService.findDistrict(districtKey, "district"); }else{ district = districtService.findDistrictById(model.getId()); - districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + if(district!=null){ + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } } if(district == null){ diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java index a12e09a99..65092281a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java @@ -18,6 +18,7 @@ public class AuctionModel extends FangjiaApiModel{ private Integer propertyType; private Integer status; private Integer limit = 10; + private String startDate; public String getId() { return id; @@ -130,4 +131,12 @@ public class AuctionModel extends FangjiaApiModel{ public void setName(String name) { this.name = name; } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } } From 7f6509a7c6345fb67aaff9e0a056a95c4a2feead Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Thu, 19 May 2022 10:18:53 +0800 Subject: [PATCH 38/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/address/AddressParserServiceImpl.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 6d9568d3d..cba023bbd 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,12 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.*; +import com.fangjia.core.util.DataUtil; +import com.fangjia.core.util.GeoUtils; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; -import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -16,17 +16,15 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors; import com.fangjia.service.address.processor.AddressProcessors; import com.fangjia.service.api.AliyunService; import com.fangjia.service.bo.AliyunBo; -import com.fangjia.service.bo.MatchAddressScore; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; -import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.bson.types.ObjectId; + import java.util.*; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; From 2cb984bbd750b596848f16e608aa54b0f9e6a0d1 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Thu, 19 May 2022 11:09:04 +0800 Subject: [PATCH 39/83] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=B4=E6=A0=8F?= =?UTF-8?q?=EF=BC=8C=E9=99=90=E5=88=B6=E5=8C=B9=E9=85=8D=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/address/AddressParserServiceImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index cba023bbd..15909a4c9 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -914,23 +914,23 @@ public class AddressParserServiceImpl implements AddressParserService { } else if (standardAddressScore > 0) { matchLevelScore = 0.8; similarityScore = standardAddressScore * SIMILARITY_RATE; - }else if(metaData.getId().equals(address.getPolygonSeaweedId())){ - matchLevelScore = 0.7; - similarityScore = 0.05; } else if(keywordScore > 0 && streetScore > 0){ - matchLevelScore = 0.6; + matchLevelScore = 0.7; similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (keywordScore > 0) { - matchLevelScore = 0.5; + matchLevelScore = 0.6; similarityScore = keywordScore * SIMILARITY_RATE; } else if (fuzzyKeywordsScore > 0) { if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) < 100){ - matchLevelScore = 0.6; + matchLevelScore = 0.7; }else{ matchLevelScore = 0.4; } similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; + }else if(metaData.getId().equals(address.getPolygonSeaweedId())){ + matchLevelScore = 0.4; + similarityScore = 0.05; } if (similarityScore > 0) { if(metaData.getBuildingType() != null){ From d546686df36d948879a086801e729b7638813071 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 19 May 2022 11:26:21 +0800 Subject: [PATCH 40/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=A5=BC=E5=B1=82?= =?UTF-8?q?=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java | 5 +++++ .../java/com/fangjia/dal/seawater/model/Seawater.java | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index e71226a4a..b2065ff21 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -203,6 +203,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i seawater.setFloor(source.getInt("floor")); }catch(Exception e){ seawater.setFloor(0); + } + try{ + seawater.setFloorInfo(source.getString("floor_info")); + }catch(Exception e){ + } if(source.containsKey("fitment")){ seawater.setFitment(source.getString("fitment")); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index c761ba76a..27b4c75b3 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity { private Double area; private Integer height; private Integer floor; + private String floorInfo; private String fitment; private Integer room; private Integer hall; @@ -195,6 +196,15 @@ public class Seawater implements SeawaterEntity { this.floor = floor; } + @Column(name = "floorInfo") + public String getFloorInfo() { + return floorInfo; + } + + public void setFloorInfo(String floorInfo) { + this.floorInfo = floorInfo; + } + @Column(name = "fitment") public String getFitment() { return fitment; From dac6321ff66a8d9ae8aeec60c0e487d86c7db978 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 19 May 2022 14:34:19 +0800 Subject: [PATCH 41/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BC=B0=E5=80=BC?= =?UTF-8?q?=E5=91=A8=E5=BD=AA=E5=9D=87=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index b62c50cd5..14563310d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1259,9 +1259,15 @@ public class PropertyController { districtDto.append("buildYear", String.valueOf(buildYear)); } districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d)); - if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){ - districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2)); + if(districtAnalyData != null){ + if (districtAnalyData.getHalfYearMarkup() != null) { + districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2)); + } + if (districtAnalyData.getAvgPrice() != null) { + districtDto.append("avgPrice", districtAnalyData.getAvgPrice()); + } } + nearCommunityDtos.add(districtDto); } Integer buildYear = districtService.getBuildYear(district); From a1d2a8139be45f0550e7b492cea4693056ac10f8 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 19 May 2022 14:57:45 +0800 Subject: [PATCH 42/83] =?UTF-8?q?=E5=8F=96=E6=B6=88=E7=B3=BB=E6=95=B0?= =?UTF-8?q?=E4=B8=8B=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 14563310d..a334cf730 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1175,11 +1175,12 @@ public class PropertyController { districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); long totalPrice = evaluation.getBigTotalPrice(); - if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ - totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); - evaluation.setBigTotalPrice(totalPrice); - evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); - } + // 取消系数下调 +// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ +// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); +// evaluation.setBigTotalPrice(totalPrice); +// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); +// } districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); List names = new ArrayList(); From b389e5b0c3ff21ea0b3bc10f48da3256474b98aa Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 19 May 2022 15:55:11 +0800 Subject: [PATCH 43/83] =?UTF-8?q?=E6=97=A0=E6=A0=B7=E6=9C=AC=E5=91=A8?= =?UTF-8?q?=E8=BE=B9=E4=BB=B7=E6=A0=BC=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/PropertyController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index a334cf730..96a414024 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1222,7 +1222,7 @@ public class PropertyController { }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); - List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null); + List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null); if(districts.size() < 2){ evaluationService.remove(evaluation.getId()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); @@ -1264,7 +1264,7 @@ public class PropertyController { if (districtAnalyData.getHalfYearMarkup() != null) { districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2)); } - if (districtAnalyData.getAvgPrice() != null) { + if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) { districtDto.append("avgPrice", districtAnalyData.getAvgPrice()); } } From 9ceacdf9a95973ac3d6b97b409dba02ad9b2ead6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Thu, 19 May 2022 18:39:13 +0800 Subject: [PATCH 44/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/seawater/dao/SeawaterDaoImpl.java | 17 +++++++++++------ .../fangjia/open/filters/AnalyticsFilter.java | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 203b6818b..b9a97f6c3 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -67,12 +67,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i must.add(addTerms("district_name", alias)); must.add(addMatch("cat", "sell")); must.add(addRange("avg_price", minPrice, maxPrice)); - - JSONObject term = new JSONObject(); - JSONObject match = new JSONObject(); - match.put("p_date", 20210501); - term.put("match", match); - must.add(term); + must.add(addRangeGte("p_date", 20210501)); bool.put("must", must); query.put("bool", bool); @@ -168,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i return term; } + private JSONObject addRangeGte(String name, Number min){ + JSONObject term = new JSONObject(); + JSONObject range = new JSONObject(); + JSONObject rangeValue = new JSONObject(); + rangeValue.put("gte", min); + range.put(name, rangeValue); + term.put("range", range); + return term; + } + private List wrapSeawater(List jsonList){ List seawaterList= new ArrayList(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java index e16664d16..37805ef21 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java @@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter { if ("1".equals(request.getParameter("__refresh_cache"))) { threadLocalData.put(ThreadLocalKey.DisableCache, "1"); } - createUID(request, response); - createSID(request, response); + //createUID(request, response); + //createSID(request, response); response.addHeader("Access-Control-Allow-Origin", "*"); response.addHeader("Access-Control-Allow-Methods", "*"); chain.doFilter(request, response); From f2d596eeb39730fd09df6e3a509685b625007761 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 20 May 2022 10:53:38 +0800 Subject: [PATCH 45/83] update --- .../data/CcbDistNameMatch.groovy | 117 +++ .../chongqing/MatchEvaluateRegionTest.groovy | 27 +- .../script/chongqing/MatchEvaluateTest.groovy | 18 +- .../fangjia/script/chongqing/PocStat.groovy | 763 ++++++++++++++++++ .../fangjia/script/chongqing/PocTest.groovy | 223 +++-- .../script/commData/CommonDatautils.groovy | 33 +- .../fangjia/script/gj/EsHousePriceStat.groovy | 20 +- .../script/gj/crawl/AnjukeAppCrawl.groovy | 198 +++++ .../script/gj/crawl/AnjukeDetailCrawl.groovy | 234 ++++-- .../gj/crawl/AnjukeDistrictMatch.groovy | 89 ++ .../fangjia/script/gj/crawl/AnjukeToDB.groovy | 628 ++++++++++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 8 +- .../script/gj/paimai/JdPaimaiCrawl.groovy | 157 ++-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 34 +- .../script/gj/paimai/PaimaiTest.groovy | 19 +- .../fangjia/script/loudong/CqfdcMatch.groovy | 35 +- .../script/thirdPriceCrawl/DataStat.groovy | 6 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 7 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- .../fangjia/script/util/OpenApiClient.groovy | 21 +- 20 files changed, 2368 insertions(+), 273 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy new file mode 100644 index 000000000..32725d092 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy @@ -0,0 +1,117 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +class CcbDistNameMatch { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new CcbDistNameMatch(); +// task.run() + task.run1() + } + + def run1(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + if(d['D'].contains("、")){ + println d['D'] + } + def arrs = d['D'].split("、") + def list = [] + def flags = [] + for(def s : arrs){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district") + if(dist){ + list << dist.name + flags << "完全一致" + }else{ + def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion) + if(jsonObject != null && jsonObject.precise!=-1) { + if(jsonObject.get("district") == newRegion){ + list << jsonObject.get("name") + flags << "匹配接口" + } + } + } + + } + if(list){ + list = list.unique() + d['E'] = list.join(",") + d['F'] = flags.join(",") + } + } + } + }) + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d -> + def regionCode = null + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if(dist){ + d['D'] = dist.name + def count = findDistrictByOracle(dist._id.toString()) + if(count && count>0){ + d['F'] = 1 + }else{ + d['H'] = dist.estate_type2 + d['J'] = dist.name_cat + } + }else{ + def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district") + if (rets) { + if(rets.size()>1){ + d['E'] = "别名-多个" + }else{ + dist = rets[0] + d['E'] = d['C'] +"--"+dist.name + def count = findAliasByOracle(dist._id.toString()) + if(count && count>0) { + d['G'] = 1 + }else{ + d['H'] = dist.estate_type2 + d['J'] = dist.name_cat + } + } + } + } + } + } + }); + } + + def findDistrictByOracle(commid){ + def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } + + def findAliasByOracle(commid){ + def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy index cd0df4d4c..b35f27015 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -23,22 +23,22 @@ class MatchEvaluateRegionTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx" def total = 1500 static main(args) { def task = new MatchEvaluateRegionTest() - task.run()//小区匹配 - task.run1()//地图匹配 +// task.run()//小区匹配 +// task.run1()//地图匹配 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 task.addPrice() - task.addArroundPrice() - task.addAuthor() +// task.addArroundPrice() +// task.addAuthor() } def run(){ ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -57,7 +57,7 @@ class MatchEvaluateRegionTest { def run1(){ ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -82,7 +82,7 @@ class MatchEvaluateRegionTest { row['N']= (int)(priceObj.avg_price * row['E']) } - def avg_price = findAvgPrice(row['M']) + def avg_price = findAvgPrice(row['F']) if(row['M'] && avg_price){ def weekendPrice = row['M'].toString().toDouble() def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) @@ -95,12 +95,12 @@ class MatchEvaluateRegionTest { def addArroundPrice(){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['A'] && !row['K']) { - def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if(!row['M']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D']) if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { def priceObj = arroundObj.result row['M'] = priceObj.unitPrice - row['N'] = (int) (priceObj.unitPrice * row['C']) + row['N'] = (int) (priceObj.unitPrice * row['E']) row['Q'] = "周边价格" } } @@ -109,21 +109,22 @@ class MatchEvaluateRegionTest { def addAuthor(){ def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] - def total = 747 def personTotal = authors.size() def avg = Math.ceil(total/personTotal) def index = 0 def count = 0; ExcelUtils.modify(path, { row -> + if(row['B']){ if(index < avg){ - row['P'] = authors[count] + row['N'] = authors[count] } index++ if(index >= avg){ count++ index = 0 } + } }); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy index 936b3b582..30997779c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -28,14 +28,14 @@ class MatchEvaluateTest { static main(args) { def task = new MatchEvaluateTest() -// task.run()//小区匹配 -// task.run1()//地图匹配 -//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 -// task.addPrice() -// task.addArroundPrice() -// task.addAuthor() + task.run()//小区匹配 + task.run1()//地图匹配 +// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 + task.addPrice() + task.addArroundPrice() + task.addAuthor() - task.stat() +// task.stat() } def addArroundPrice(){ @@ -57,7 +57,7 @@ class MatchEvaluateTest { // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['D'] && row['B']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false) if(jsonObject != null && jsonObject.precise!=-1) { row['D'] = jsonObject.get("id") row['E'] = jsonObject.get("city") @@ -77,7 +77,7 @@ class MatchEvaluateTest { // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['D'] && row['B']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true) if(jsonObject != null && jsonObject.precise!=-1) { row['D'] = jsonObject.get("id") row['E'] = jsonObject.get("city") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy new file mode 100644 index 000000000..8a4c43d95 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy @@ -0,0 +1,763 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PocStat { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + + static main(args) { + def task = new PocStat(); + task.regionDistStat(); +// task.sampleStat() +// task.tradeStat() +// task.paimaiStat() +// task.evalStat() +// +// task.fileldStat() + } + + def regionDistStat(){ + def city = "重庆" + def regionMap = CommonDatautils.getRegionsByCity(city) + def set =regionMap.keySet(); + def rows = [] + rows << ["区域","小区量"] + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"]) + for(def region : set){ + def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district") + + def regionCount = 0 + def regionHouseLDCount = 0 + for(def d : dists){ + if(d.region == region){ + def loudongCount = CommonDatautils.findLoudong(d._id.toString()) + if(loudongCount && loudongCount>0){ + regionCount = regionCount+ loudongCount + } + + def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString()) + if(houseLouDongCount && houseLouDongCount>0){ + regionHouseLDCount = regionHouseLDCount + houseLouDongCount + } + } + } + rows << [region, count, regionCount, regionHouseLDCount] + println region +" 完成" + } + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计") + } + + def sampleStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zHLevel1 = new AtomicLong() + def zHLevel2 = new AtomicLong() + def zHLevel3 = new AtomicLong() + def zHLevel4 = new AtomicLong() + def zHLevel5 = new AtomicLong() + + def fHLevel1 = new AtomicLong() + def fHLevel2 = new AtomicLong() + def fHLevel3 = new AtomicLong() + def fHLevel4 = new AtomicLong() + def fHLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zHLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fHLevel2.addAndGet(1) + }else if(results.size()<20){ + fHLevel3.addAndGet(1) + }else if(results.size()<50){ + fHLevel4.addAndGet(1) + }else if(results.size()>=50){ + fHLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zHLevel1.addAndGet(1) + + println d.city+"#"+d.region+"#"+d.name + }else{ + fCityDistTotal.addAndGet(1) + fHLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本") + } + + + def tradeStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zTLevel1 = new AtomicLong() + def zTLevel2 = new AtomicLong() + def zTLevel3 = new AtomicLong() + def zTLevel4 = new AtomicLong() + def zTLevel5 = new AtomicLong() + + def fTLevel1 = new AtomicLong() + def fTLevel2 = new AtomicLong() + def fTLevel3 = new AtomicLong() + def fTLevel4 = new AtomicLong() + def fTLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zTLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fTLevel2.addAndGet(1) + }else if(results.size()<20){ + fTLevel3.addAndGet(1) + }else if(results.size()<50){ + fTLevel4.addAndGet(1) + }else if(results.size()>=50){ + fTLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zTLevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fTLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交") + println "成交案例统计完成" + } + + def paimaiStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zPLevel1 = new AtomicLong() + def zPLevel2 = new AtomicLong() + def zPLevel3 = new AtomicLong() + def zPLevel4 = new AtomicLong() + def zPLevel5 = new AtomicLong() + + def fPLevel1 = new AtomicLong() + def fPLevel2 = new AtomicLong() + def fPLevel3 = new AtomicLong() + def fPLevel4 = new AtomicLong() + def fPLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zPLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fPLevel2.addAndGet(1) + }else if(results.size()<20){ + fPLevel3.addAndGet(1) + }else if(results.size()<50){ + fPLevel4.addAndGet(1) + }else if(results.size()>=50){ + fPLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zPLevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fPLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖") + println "拍卖案例统计完成" + } + def evalStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zELevel1 = new AtomicLong() + def zELevel2 = new AtomicLong() + def zELevel3 = new AtomicLong() + def zELevel4 = new AtomicLong() + def zELevel5 = new AtomicLong() + + def fELevel1 = new AtomicLong() + def fELevel2 = new AtomicLong() + def fELevel3 = new AtomicLong() + def fELevel4 = new AtomicLong() + def fELevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zELevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fELevel2.addAndGet(1) + }else if(results.size()<20){ + fELevel3.addAndGet(1) + }else if(results.size()<50){ + fELevel4.addAndGet(1) + }else if(results.size()>=50){ + fELevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zELevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fELevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估") + println "评估案例统计完成" + } + + def fileldStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + final CountDownLatch latch = new CountDownLatch(dists.size()) + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def alias_count = new AtomicLong() + def address_count = new AtomicLong() + def block_count = new AtomicLong() + def township_count = new AtomicLong() + def price_count = new AtomicLong() + def lng_count = new AtomicLong() + def finishBuildTime_count = new AtomicLong() + def developer_count = new AtomicLong() + def green_count = new AtomicLong() + def ratio_count = new AtomicLong() + def carsite_count = new AtomicLong() + def propertyManagement_count = new AtomicLong() + def managementFee_count = new AtomicLong() + def building_count = new AtomicLong() + def holed_count = new AtomicLong() + def car_holed_count = new AtomicLong() + def all_area_count = new AtomicLong() + def key_school_count = new AtomicLong() + def school_count = new AtomicLong() + def traffic_count = new AtomicLong() + def market_count = new AtomicLong() + def hospital_count = new AtomicLong() + def bank_count = new AtomicLong() + + def f_alias_count = new AtomicLong() + def f_address_count = new AtomicLong() + def f_block_count = new AtomicLong() + def f_township_count = new AtomicLong() + def f_price_count = new AtomicLong() + def f_lng_count = new AtomicLong() + def f_finishBuildTime_count = new AtomicLong() + def f_developer_count = new AtomicLong() + def f_green_count = new AtomicLong() + def f_ratio_count = new AtomicLong() + def f_carsite_count = new AtomicLong() + def f_propertyManagement_count = new AtomicLong() + def f_managementFee_count = new AtomicLong() + def f_building_count = new AtomicLong() + def f_holed_count = new AtomicLong() + def f_car_holed_count = new AtomicLong() + def f_all_area_count = new AtomicLong() + def f_key_school_count = new AtomicLong() + def f_school_count = new AtomicLong() + def f_traffic_count = new AtomicLong() + def f_market_count = new AtomicLong() + def f_hospital_count = new AtomicLong() + def f_bank_count = new AtomicLong() + + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + def index = new AtomicLong() + for(def i=0;i - if(!row['P'] && row['C']){ - def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null) + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null) if(jsonObject != null && jsonObject.precise!=-1) { - row['P'] = jsonObject.get("name") - row['Q'] = jsonObject.get("id") + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("id") + } + } + }) + } + + def matchMap(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + def propertyType = jsonObject.get("propertyType") + if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){ + row['Q'] = jsonObject.get("name") + row['R'] = new ObjectId().toString() + row['W'] = propertyType + + def arroundObj = FangjiaClientUtils.arroundPrice(city, address) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['F'] = priceObj.unitPrice + } + } + } } }) @@ -44,34 +93,51 @@ class PocTest { def addPrice(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['Q']){ - def dist = CommonDatautils.findDistrictById(row['Q']) + def commid = row['R'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid) if(dist){ def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) if(priceObj){ - row['E'] = priceObj.avg_price + row['F'] = priceObj.avg_price } } } }) } + def addArroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['F'] = priceObj.unitPrice + row['W'] = "周边均价" + } + } + }); + } + def standard(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(row['C']){ - def jsonObject = OpenApiClient.standard2Action(row['A'], row['C']) + def city = row['B'] + def address = row['D'] + if(address){ + println "[standard] address:"+address + def jsonObject = OpenApiClient.standard2Action(city, address) if(jsonObject){ println jsonObject - row['F'] = jsonObject.get("province") - row['G'] = jsonObject.get("city") - row['H'] = jsonObject.get("district") - row['I'] = jsonObject.get("township") - row['J'] = jsonObject.get("street") - row['K'] = jsonObject.get("streetNumber") - row['L'] = jsonObject.get("buildingNumber") - row['M'] = jsonObject.get("unitNumber") - row['N'] = jsonObject.get("floor") - row['O'] = jsonObject.get("roomNumber") + row['G'] = jsonObject.get("province") + row['H'] = jsonObject.get("city") + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("township") + row['K'] = jsonObject.get("street") + row['L'] = jsonObject.get("streetNumber") + row['M'] = jsonObject.get("buildingNumber") + row['N'] = jsonObject.get("unitNumber") + row['O'] = jsonObject.get("floor") + row['P'] = jsonObject.get("roomNumber") } } @@ -81,23 +147,27 @@ class PocTest { def addLoudongNum(path){ ExcelUtils.multiThreadingModify(path, { row -> - if (row['Q'] && row['L']) { - def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + def commid = row['R'] + def houseNum = row['M'] + def unitNum = row['N'] + println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum + if (commid && houseNum) { + def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum) if (jsonObject && jsonObject.result) { if(jsonObject.result.size()>=1){ def obj = jsonObject.result[0] - row['S'] = obj.id - row['R'] = obj.houseNumber + row['T'] = obj.id + row['S'] = obj.houseNumber } } - if(row['O']){ - def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if(row['P']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P']) if (jsonObject1 && jsonObject1.result) { - if(jsonObject.result.size()>=1){ - def obj = jsonObject.result[0] - row['U'] = obj.id - row['T'] = obj.roomNum + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + row['V'] = obj.id + row['U'] = obj.roomNum } } } @@ -111,12 +181,12 @@ class PocTest { rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"] for(def row : datas){ def data = [] - data << row['A'] data << row['B'] data << row['C'] data << row['D'] - if (row['Q'] && row['L']) { - def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + data << row['E'] + if (row['R'] && row['M']) { + def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N']) if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) { def obj = jsonObject.result[0] data << obj.houseNumber @@ -144,8 +214,8 @@ class PocTest { data << ""//景观 } - if(row['O']){ - def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if(row['P']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P']) if (jsonObject1 && jsonObject1.result) { if(jsonObject1.result.size()>=1){ def obj = jsonObject1.result[0] @@ -174,18 +244,20 @@ class PocTest { def rows = [] rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"] for(def d : datas){ - if(d['Q']){ - def dist = CommonDatautils.findDistrictById(d['Q']) + def commid = d['R'] + println "[addDetail] commid:"+commid + if(commid){ + def dist = CommonDatautils.findDistrictById(commid) if(dist){ def row = [] def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name) if(jsonObject) { - row << d['A'] row << d['B'] row << d['C'] + row << d['D'] row << jsonObject.get("name")//小区名称 - row << d['Q']//小区ID + row << commid//小区ID if(jsonObject.get("alias")){ row << jsonObject.get("alias").join(",").replace('"',"")//小区别名 }else{ @@ -207,7 +279,7 @@ class PocTest { row << jsonObject.get("buildingCount")//住宅幢数 row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数 row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比********* - row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积 + row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积 } @@ -271,28 +343,35 @@ class PocTest { rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"] def keys = [] for(def d : datas){ - if(d['Q']) { - def dist = CommonDatautils.findDistrictById(d['Q']) + def commid = d['R'] + println "[createPaimai] commid:"+commid + if(commid) { + def dist = CommonDatautils.findDistrictById(commid) if (dist) { def key = dist.city+"#"+dist.region+"#"+dist.name if(keys.contains(key)){ continue } keys << key - def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10) + def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100) if(jsonObject && jsonObject.result){ def results = jsonObject.result for(def r: results){ def row = [] - row << d['A'] row << d['B'] row << d['C'] + row << d['D'] row << r.matchName row << r.address row << r.houseNum row << r.roomNum row << r.area - row << r.startDate + if(r.startDate){ + row << r.startDate.substring(0, r.startDate.indexOf("T")) + }else{ + row << "" + } + row << r.count row << r.status row << r.tradeAvgPrice @@ -314,13 +393,15 @@ class PocTest { rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] def keys = [] for(def d : datas) { - if (d['Q']) { + def commid = d['R'] +// println "[createHouse] commid:"+commid + if (commid) { def key = d['Q'] if (keys.contains(key)) { continue } keys << key - def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20) + def jsonObject = OpenApiClient.houseList(commid,"20210501",20) if(jsonObject && jsonObject.result) { def results = jsonObject.result for (def r : results) { @@ -368,7 +449,7 @@ class PocTest { rows << row } - println d['Q'] +" "+results.size() + println d['R'] +" "+results.size() } } } @@ -381,13 +462,14 @@ class PocTest { rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] def keys = [] for(def d : datas) { - if (d['Q']) { - def key = d['Q'] - if (keys.contains(key)) { + def commid= d['R'] + println "[createEvaluation] commid:"+commid + if (commid) { + if (keys.contains(commid)) { continue } - keys << key - def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20) + keys << commid + def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20) if(jsonObject && jsonObject.result) { def results = jsonObject.result for (def r : results) { @@ -432,7 +514,7 @@ class PocTest { rows << row } - println d['Q'] +" "+results.size() + println d['R'] +" "+results.size() } } } @@ -446,13 +528,14 @@ class PocTest { rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] def keys = [] for(def d : datas) { - if (d['Q']) { - def key = d['Q'] - if (keys.contains(key)) { + def commid= d['R'] + println "[createDealPrice] commid:"+commid + if (commid) { + if (keys.contains(commid)) { continue } - keys << key - def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20) + keys << commid + def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20) if(jsonObject && jsonObject.result) { def results = jsonObject.result for (def r : results) { @@ -501,7 +584,7 @@ class PocTest { rows << row } - println d['Q'] +" "+results.size() + println d['R'] +" "+results.size() } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d6ab63366..26c6a3e15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -68,6 +68,10 @@ class CommonDatautils { public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl) + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl) + + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -704,7 +708,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","吴中路9弄小区") +// q.put("name","森望芳草绿岸") DBObject keys = new BasicDBObject(); if(fields){ @@ -827,6 +831,8 @@ class CommonDatautils { keys.put("out_pics", 1); keys.put("lat", 1); keys.put("lng", 1); + keys.put("lat2", 1); + keys.put("lng2", 1); keys.put("name_cat", 1); keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) @@ -1053,9 +1059,16 @@ class CommonDatautils { public static def queryDistrictById(def id){ + def dbo = null DBObject q = new BasicDBObject(); - q.put("_id", new ObjectId(id)); - def dbo = getDataStoreConnection().findOne(q); + try{ + q.put("_id", new ObjectId(id.trim().toString())); + dbo = getDataStoreConnection().findOne(q); + }catch(Exception e){ + println id + } + + return dbo; } @@ -3184,5 +3197,19 @@ class CommonDatautils { def avg_radios = (int)(radios.sum()/radios.size()) return avg_radios; } + + static def findLoudong(commid){ + def q = new BasicDBObject() + q.put("id_in_seaweed", commid) + def count = LOUDONG_LINE_COLL.count(q) + return count; + } + + static def findHouseLoudong(commid){ + def q = new BasicDBObject() + q.put("district_id", commid) + def count = HOUSE_LOUDONG_COLL.count(q) + return count; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy index 4628a207b..d5123668b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy @@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{ def run(){ def cityMap = CommonDatautils.getAllCityRegions() cityMap.each{city,regions-> - findPriceByCity(city, regions); + + findPriceByCity(city, regions); } } @@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{ q.put("region" , r.region) q.put("month", r.month) def mom = null - if(lastObj){ - mom = (r.price -lastObj.avg_price)/lastObj.avg_price - mom = DataHandleUtils.getDoubleRound(4, mom) + if(lastObj && lastObj.avg_price && r.price){ + mom = (r.price -lastObj.avg_price)/lastObj.avg_price + mom = DataHandleUtils.getDoubleRound(4, mom) } if(obj){ if(obj.avg_price != r.price){ @@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{ def findEsAvgPrice(city, region){ def detail = [:] - detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1]) + detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0]) detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]]) def must = [] @@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{ if(!priceValue){ priceValue = row['1-metric'] } - if(priceValue){ + if(priceValue && priceValue.value){ price = priceValue.value.toInteger() } result.price = price; def date = row.key_as_string.replace("-","").substring(0, 6) def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM") if(date!=curMonth){ - result.month = date.toInteger() - rets << result + if(date.toInteger()>200001 && result.price){ + result.month = date.toInteger() + rets << result + } + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy new file mode 100644 index 000000000..4601c3c60 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -0,0 +1,198 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeAppCrawl { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeAppCrawl(); + task.run(); +// task.run2() + } + + def run2(){ + def url = "https://m.anjuke.com/fz/community/" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def links = getRegion(url, driver) + for(def link : links){ + def doc = getDocument(url, driver) + def str = doc.toString().substring(doc.toString().indexOf("")) + if(str){ + str = str.substring(str.indexOf("} ("), str.indexOf("));")) + def arrs = str.split(",") + pritln arrs.size() + } + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + + def getRegion(url, driver){ + //def doc = getDocument(url, driver) + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") + header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") + header.put("referer","https://m.anjuke.com/fz/") + header.put("sec-fetch-dest","empty") + header.put("sec-fetch-mode","cors") + header.put("sec-fetch-site","same-origin") + header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1") + header.put("sw8-correlation","c291cmNl:YmVpZG91") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1") + + def doc = ABuYunUtil.doGet(url, null, header, "utf-8") + def links = [] + if(doc){ + def eles = JQueryUtils.select(".column2 a", doc) + for(def e : eles){ + def href = e.attr("href") + links << href + } + } + return links; + } + + + + + def run(){ + //"仓山":4071, + def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384] + map.each{region, area_id-> + for(def i=1;i<40;i++){ + def stop = false; + def all_stop = false + while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380 + //城口 + def flagMap = getInfo(stop, i, area_id) + stop = flagMap.stop + all_stop = flagMap.all_stop + } + if(all_stop){ + break; + } + } + } + } + + def getInfo(def stop, def i, def area_id){ + + def map = [:] + def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1" + + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") + header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") + header.put("referer","https://m.anjuke.com/fz/community/") + header.put("sec-fetch-dest","empty") + header.put("sec-fetch-mode","cors") + header.put("sec-fetch-site","same-origin") + header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1") + header.put("sw8-correlation","c291cmNl:YmVpZG91") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1") + + def ret = ABuYunUtil.doGet(url, null, null, "utf-8") + def all_stop = false; + if (ret) { + JSONObject jsonObj = JSONObject.fromObject(ret); + if(jsonObj && jsonObj.status=="ok" ){ + def data = jsonObj.data + if(data && data.total.toInteger()>0){ + def communities = data.communities + + println "【${i}】 "+communities.size() +" "+url + if(communities.size()>0){ + save("福州", communities) + Thread.sleep(10000) + stop = true; + }else{ + stop = true; + println url +" 返回0" + all_stop = true; + } + + }else{ + stop = false; + println url +" 无数据" + } + + + }else{ + stop = false; + Thread.sleep(5000) + } + } + + map.stop = stop + map.all_stop = all_stop; + return map; + } + + def save(city, def ret){ + def index = 0 + for(def o : ret){ + index++ + def obj = findIsExist(o.jumpAction) + if(!obj){ + def udate = new BasicDBObject() + o.each{k,v-> + udate.put(k,v) + } + udate.put("city", city) + udate.put("c_date", new Date()) + detail.save(udate) + println index+" [保存]"+o.jumpAction + }else{ + println index+" [已存在]"+o.jumpAction + } + + } + } + + def findIsExist(def url){ + def q = new BasicDBObject() + q.put("jumpAction",url) + return detail.findOne(q); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy index d4f3001b3..ab7aa4278 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy @@ -11,9 +11,15 @@ import org.openqa.selenium.By import org.openqa.selenium.WebDriver import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.remote.DesiredCapabilities +import org.openqa.selenium.remote.RemoteWebDriver class AnjukeDetailCrawl { + DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") + + DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url") + static main(args) { def task = new AnjukeDetailCrawl(); task.run(); @@ -21,9 +27,15 @@ class AnjukeDetailCrawl { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - def url = "https://chongqing.anjuke.com/community/?from=navigation" + def url = "https://chongqing.anjuke.com/" WebDriver driver = new ChromeDriver() +// DesiredCapabilities capabilities = DesiredCapabilities.chrome(); +// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL +// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities); + + driver.manage().deleteAllCookies() def regions = getRegions(driver, url) + println "区域链接: "+regions.size() getAllRegions(driver, regions) } @@ -31,44 +43,88 @@ class AnjukeDetailCrawl { def getAllRegions(driver, regions){ for(def url : regions){ - driver.get(url) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = getDocument(url, driver) def total = getTotal(doc) if(total) { if (total > 1250) { - + def links = getMoreLinks(url, driver) + println url +" "+links.size() + saveDetailLinks(links) +// parse(links,driver) } else { def links = getAllLinks(url, driver) - parse(links,driver) + println url +" "+links.size() + saveDetailLinks(links) +// parse(links,driver) } } } } + def saveDetailLinks(links){ + for(def url : links){ + def data = new BasicDBObject() + data.put("city","重庆") + data.put("url", url) + data.put("source", "安居客") + data.put("c_date", new Date()) + urlColl.save(data) + } + } + + def getMoreLinks(url, driver){ + def doc = Jsoup.parse(driver.getPageSource()); + def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc) + def links = [] + if(pricelinks){ + for(def link : pricelinks){ + if(link.text()!="不限"){ + def urlLink = link.attr("href") + getDocument(urlLink, driver) +// driver.get(urlLink) + links.addAll(getAllLinks(urlLink, driver)) + println urlLink +" "+links.size() + } + } + } + return links; + } + def getAllLinks(url, driver){ def stop = false def urls = [] - while(!stop) { - def doc = Jsoup.parse(driver.getPageSource()); + def doc = Jsoup.parse(driver.getPageSource()); + def total = getTotal(doc) + def page = getPage(total, 25) + def pages = [] + for(def i=1;i<=page;i++){ + pages << url.substring(0, url.size()-1)+"-p"+i+"/" + } + for(def pageUrl : pages){ + doc = getDocument(pageUrl, driver) def details = JQueryUtils.select(".list-cell a.li-row", doc) - for (def d : details) { - urls << d.attr("href") - } - def nextEle = JQueryUtils.select(".next", doc) - if (nextEle) { - driver.findElement(By.cssSelector(".next")).click() - }else{ - stop = true - } - if(urls.size()>20){ - stop = true + def link = d.attr("href") + if(!urls.contains(link)){ + urls << d.attr("href") + } } } return urls } + def getPage(def total, def pageSize){ + if(total && total.toInteger()>0){ + def page = total.toInteger()/pageSize; + if(page>(int)page){ + page = (int)page + 1 + return page + } + return (int)page; + } + return null + } def getTotal(doc){ def totalText = JQueryUtils.selectText(".total-info", doc) @@ -82,9 +138,19 @@ class AnjukeDetailCrawl { } def getRegions(driver, url){ +// def doc = getDocument(url, driver) driver.get(url) - Thread.sleep(3000) def doc = Jsoup.parse(driver.getPageSource()); + def els = JQueryUtils.select("a:contains(小区)", doc) + if(els){ + def href = els[0].attr("href") + doc = Jsoup.parse(driver.getPageSource()); + // driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click() +// driver.findElement(By.linkText("小区")).click() + doc = getDocument(href, driver) + } + + // def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) // if(loginIfm){ // def href = loginIfm[0].attr("src") @@ -107,23 +173,35 @@ class AnjukeDetailCrawl { def regionEles = JQueryUtils.select(".region-parents li a", doc) def urls = [] for(def e : regionEles){ - urls << e.attr("href") + if(e.text()!="不限"){ + urls << e.attr("href") + } } return urls; } + def getNontCheckPage(url, driver, doc){ + def stop = false; + while(!stop){ + if(doc){ + + } + } + return doc; + } + def parse(def links, driver){ def rets = [] for(def link : links){ def district = parsing(link, driver) if(district){ - rets << district + // rets << district - if (rets.size() % 100 == 0) { - def add= saveData(rets) - println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - rets = [] - } + // if (rets.size() % 100 == 0) { + saveObj(district) + println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + // rets = [] + // } } } if (rets.size()> 0) { @@ -136,33 +214,61 @@ class AnjukeDetailCrawl { def saveData(def results){ def add = 0 for(def ret : results){ - def q = new BasicDBObject() - q.put("url", ret.url) - def dbo = detailColl.findOne(q); - - DBObject obj = new BasicDBObject(); - ret.each{k,v-> - obj.put(k,v) - } - - if(dbo){ - obj.m_date = new Date() - detailColl.update(q, new BasicDBObject('$set', obj)) - }else{ - add++ - obj.c_date = new Date() - obj.m_date = new Date() - try{ - detailColl.save(obj) - }catch(Exception e){ - println obj.url - } - } + saveObj(ret) } - return add } + def saveObj(def ret){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = detailColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + detailColl.update(q, new BasicDBObject('$set', obj)) + }else{ + + obj.c_date = new Date() + obj.m_date = new Date() + try{ + detailColl.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + def parsing(url, driver){ @@ -191,10 +297,9 @@ class AnjukeDetailCrawl { def infoDiv = JQueryUtils.select(".info-list", doc) district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) - district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) - district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) - district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) - + district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text()) + district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) + district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() if (houseCountStr) { houseCountStr = houseCountStr.replace("户", "") @@ -202,9 +307,13 @@ class AnjukeDetailCrawl { district.house_count = houseCountStr.toInteger() } } - - district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) + district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) + district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text()) + + district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) + district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text()) + def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) if(carSiteStr){ if(carSiteStr.contains("(")){ @@ -214,16 +323,19 @@ class AnjukeDetailCrawl { district.car_site = carSiteStr } } - district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) - district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) - district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) - district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) - district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text() + district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) + + district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text()) + + district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) + district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) district.url = url district.c_date = new Date() district.m_date = new Date() + district.crawl_type = "chrome" district.root = "anjuke.com" - stop = true; }else{ Thread.sleep(3000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy new file mode 100644 index 000000000..f990711be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class AnjukeDistrictMatch { + + static main(args) { + def task = new AnjukeDistrictMatch(); + task.run(); +// task.run1() + } + + def run1(){ + def regionPointsMap = findRegionRange("重庆") + def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + } + def run(){ + + + + def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx" + ExcelUtils.multiThreadingModify(filePath,{row-> + if(row['A'] && !row['J']){ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def address = row['E'] +// def flag = "别名匹配" + + +// def dist = matchDist(city, null, name) +// if(!row['J'] && name.contains("(")){ +// name = name.replace("(${region})","").replace("(","").replace(")","") +// flag = "去括号匹配" +// dist = matchDist(city, region, name) +// } +// if(!dist){ +// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") +// flag = "映射别名匹配" +// } +// if(!dist) { +// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") +// if(rets && rets.size()==1){ +// dist = rets[0] +// } +// } + +// +// if(!dist){ +// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district") +// flag = "地址匹配" +// if(dists.size()>1){ +// println address +" 地址相同:"+dists.size() +// }else{ +// if(dists.size()==1){ +// dist = dists[0] +// } +// } +// } + +// if(dist){ +// row['J'] = flag +// row['K'] = dist._id.toString() +// row['L'] = dist.region +// row['M'] = dist.name +// row['N'] = dist.address +// } + + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) + if(jsonObject != null && jsonObject.precise!=-1) { +// row['J'] = "匹配接口" +// row['K'] = jsonObject.get("id") +// row['L'] = jsonObject.get("district") +// row['M'] = jsonObject.get("name") +// row['N'] = jsonObject.get("address") + row['M'] = jsonObject.get("name") + } + } + }) + } + + def matchDist(city, region, name){ + def dist = CommonDatautils.findDistrict(city, region, name, "district") + return dist; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy new file mode 100644 index 000000000..1954d135c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy @@ -0,0 +1,628 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +class AnjukeToDB { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeToDB(); +// task.run(); +// task.addNoMatch() + task.updateEstateType() +// task.gaodeRangeData() + } + + def updateEstateType(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1) + for(def row : datas){ + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj) { + def type = getEstatType(obj.extend.type) + def commId = row['L'] + if(!commId){ + commId = row['G'] + } + if(type){ + if(type=="别墅"){ + def udata = new BasicDBObject() + udata.put("estate_type2", type) + udata.put("m_date", new Date()) + println commId + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata)) + } + }else{ +// def udata = new BasicDBObject() +// udata.put("status", 1) +// udata.put("name_cat",11) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata)) + + } + } + } + } + + def gaodeRangeData(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1) + def index = 0 + for(def d : datas){ + index++ + println index + def personName = d['AB'] + if(personName){ + def commid = d['J'] + if(commid){ + def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district") + if(obj){ + updateDist(d, obj._id.toString()) + } + }else{ + saveGaode(d) + } + }else{ + def commid = d['J'] + updateDist(d, commid) + } + + } + } + + def saveGaode(d){ + def oldName = d['E'] + def distName = nameFileter(d['C'], oldName) + def udata = new BasicDBObject() + def alias = [] + + def aliasStr = d['F'] + if(aliasStr){ + def arrs = aliasStr.split("/") + for(def s : arrs){ + alias << s + } + } + if(alias.size()>0){ + udata.put("alias", alias) + } + + def address = AddressUtil.formatAddress("重庆", d['G']) + if(address){ + udata.put("address", address) + } + if(d['N']){ + udata.put("virescence_percent", d['N']) + } + + if(d['O']){ + udata.put("estate_charge", d['O']) + } + if(d['Q']){ + udata.put("cubage_percent", d['Q']) + } + if(d['R']){ + udata.put("developer", d['R']) + } + + if( d['S']){ + udata.put("build_area", d['S']) + } + + if(d['T']){ + def t = d['T'].substring(0, 3) + udata.put("build_date", t) + } + + if(d['V'] && d['V'].isNumber()){ + udata.put("car_site_count", d['V'].toInteger()) + } + + if(d['Y']){ + def build_type = getBuildType2(d['Y']) + if(build_type){ + udata.put("build_type", build_type) + } + } + + if(d['Z']){ + udata.put("property_life", d['Z']) + } + + def lat = null + def lng = null + if(d['H']){ + lat = d['H'] + lng = d['I'] + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + Long dlat = lat.toDouble() * 10000000000L + Long dlng = lng.toDouble() * 10000000000L + udata.put("lat", dlat) + udata.put("lng", dlng) + } + def type = "普通住宅" + if(distName.contains("别墅")) { + type = "别墅" + }else if(distName.contains("商住")){ + type = "商住" + } + + if(udata){ + udata.put("name_cat",0) + udata.put("status",0) + udata.put("cat","district") + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("city", "重庆") + udata.put("region",d['C']) + udata.put("name", distName) + udata.put("estate_type2",type) + + def markMap = [:] + markMap.put("data_source", "机器入库") + udata.put("mark_map",markMap) + DbUtils.seaweed.save(udata); + + def stop = false + while (!stop){ + if(udata._id.toString()){ + addPrice(udata) + stop = true; + } + } + } + } + + def updateDist(d, commid){ + + if(commid){ + def dist = CommonDatautils.queryDistrictById(commid) + if(dist){ + def oldName = d['E'] + def distName = nameFileter(d['C'], oldName) + def udata = new BasicDBObject() + if(!distName.contains("号")){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(!alias.contains(distName) && dist.name!=distName){ + alias << distName + } + def aliasStr = d['F'] + if(aliasStr){ + def arrs = aliasStr.split("/") + for(def s : arrs){ + alias << s + } + } + if(alias.size()>0 && alias!=dist.alias){ + udata.put("alias", alias) + } + } + + if(dist.address && !dist.address.contains("号")){ + def address = AddressUtil.formatAddress("重庆", d['G']) + if(address){ + udata.put("address", address) + } + } + + if(!dist.virescence_percent && d['N']){ + udata.put("virescence_percent", d['N']) + } + + if(!dist.estate_charge && d['O']){ + udata.put("estate_charge", d['O']) + } + if(!dist.cubage_percent && d['Q']){ + udata.put("cubage_percent", d['Q']) + } + if(!dist.developer && d['R']){ + udata.put("developer", d['R']) + } + + if(!dist.build_area && d['S']){ + udata.put("build_area", d['S']) + } + + if(!dist.build_date && d['T']){ + if(d['T'].size()>=4){ + def t = d['T'].substring(0, 4) + udata.put("build_date", t) + } + + } + + if(!dist.car_site_count && d['V'] && d['V'].isNumber()){ + udata.put("car_site_count", d['V'].toInteger()) + } + + if(!dist.build_type && d['Y']){ + def build_type = getBuildType2(d['Y']) + if(build_type){ + udata.put("build_type", build_type) + } + } + + if(!dist.property_life && d['Z']){ + udata.put("property_life", d['Z']) + } + + def lat = null + def lng = null + if(d['H']){ + lat = d['H'] + lng = d['I'] + + def slat = dist.lat/10000000000.0 + def slng = dist.lng/10000000000.0 + + def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat); + if(distance>1000){ + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + Long dlat = lat.toDouble() * 10000000000L + Long dlng = lng.toDouble() * 10000000000L + udata.put("lat", dlat) + udata.put("lng", dlng) + } + + } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改] ${dist.name}"+ udata + } + } + } + } + + def addNoMatch(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1) + for(def d : datas){ + add(d) + } + } + def run(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1) + for(def data : datas){ + if(data['K']=="芝芝"){ + def flag = data['M'] + if(flag && flag=="新增"){ + add(data) + }else{ + update(data) + } + } + } + } + + + def add(def row){ + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj){ + def type = getEstatType(obj.extend.type) + if(type){ + def city = obj.base.cityName + def region = obj.base.areaName + def lat2 = null + def lng2 = null + def lat = null + def lng = null + if(obj.base.lat){ + lat2 = obj.base.lat + lng2 = obj.base.lng + + JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu") + if(convertObj.getInt("status") == 1) { + String lo = convertObj.get("locations") + lng = Double.parseDouble(lo.split(",")[0]) + lat = Double.parseDouble(lo.split(",")[1]) + } + } + + def distName = nameFileter(obj.base.areaName, obj.base.name) + + def buildTypeStr = getBuildType(obj.base.buildTypeStr) + + def address = AddressUtil.formatAddress(obj.base.city, obj.base.address) + + def dist = CommonDatautils.findDistrict(city, region, distName, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",region) + data.put("name", distName) + data.put("estate_type2",type) + if(address){ + data.put("address", obj.base.address) + } + + if(lat){ + Long alat = lat*10000000000L + Long alng = lng*10000000000L + data.put("lat", alat) + data.put("lng", alng) + Long alat2 = lat2.toDouble()*10000000000L + Long alng2 = lng2.toDouble()*10000000000L + data.put("lat2",alat2) + data.put("lng2",alng2) + } + data.put("finish_building_date",obj.extend.completionTime) + if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){ + data.put("household_count",obj.extend.totalHouseHoldNum.toInteger()) + } + data.put("build_area", obj.extend.totalArea) + + data.put("virescence_percent", obj.extend.landscapingRatio) + data.put("build_type", buildTypeStr) + data.put("business_quarter",obj.base.tradingAreaName) + + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + DbUtils.seaweed.save(data); + + def stop = false + while (!stop){ + if(data._id.toString()){ + addPrice(data) + stop = true; + } + } + + } + }else{ + println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type + } + }else{ + println anjukeId+" 没找到" + } + + } + def update(def row){ + def commId = row['L'] + if(!commId){ + commId = row['G'] + } + + def dist = CommonDatautils.queryDistrictById(commId) + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj && dist){ + def udata = new BasicDBObject() + + def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address) + if(address){ + if(!dist.address){ + if(obj.base.address){ + udata.put("address",obj.base.address) + } + }else{ + if(dist.address.contains("号")){ + if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){ + def addr_alias = dist.addr_alias + if(!addr_alias){ + addr_alias = [] + } + if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){ + addr_alias << obj.base.address + } + if(addr_alias!=dist.addr_alias && addr_alias.size()>0){ + udata.put("addr_alias", addr_alias) + } + } + }else{ + udata.put("address",obj.base.address) + } + } + } + + def distName = nameFileter(obj.base.areaName, obj.base.name) + if(!distName.contains("号")){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(alias.contains(distName)){ + alias << distName + } + if(alias.size()>0 && alias!=dist.alias){ + udata.put("alias", alias) + } + } + + if(!dist.build_type){ + def buildTypeStr = getBuildType(obj.base.buildTypeStr) + if(buildTypeStr){ + udata.put("build_type",buildTypeStr) + } + + } + if(!dist.household_count){ + if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){ + udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger()) + } + } + if(!dist.finish_building_date){ + if(obj.extend.completionTime){ + udata.put("finish_building_date",obj.extend.completionTime) + } + } + + if(!dist.virescence_percent){ + if(obj.extend.landscapingRatio){ + udata.put("virescence_percent", obj.extend.landscapingRatio) + } + } + if(!dist.business_quarter){ + if(obj.base.tradingAreaName){ + udata.put("business_quarter",obj.base.tradingAreaName) + } + } + def lat2 = null + def lng2 = null + if(dist.lat2){ + lat2 = dist.lat2/10000000000.0 + lng2 = dist.lng2/10000000000.0 + } + if(dist.lat && !lat2){ + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3) + lng2 = baidu.x + lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + } + + if(obj.base.lat) { + def alat2 = obj.base.lat.toDouble() + def alng2 = obj.base.lng.toDouble() + def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2); + if(distance>1000){ + println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................" + + def lng = null + def lat = null + JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu") + if(convertObj.getInt("status") == 1) { + String lo = convertObj.get("locations") + lng = Double.parseDouble(lo.split(",")[0]) + lat = Double.parseDouble(lo.split(",")[1]) + } + + Long blat2 = alat2 *10000000000L + Long blng2 = alng2 *10000000000L + Long gLat2 = lat *10000000000L + Long gLng2 = lng *10000000000L + + udata.put("lat2", blat2) + udata.put("lng2", blng2) + udata.put("lat", gLat2) + udata.put("lng", gLng2) + } + } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改] ${dist.name}"+ udata + } + + } + } + + def findAnjukeDistrictDetail(anjukeId){ + def q = new BasicDBObject() + q.put("_id",new ObjectId(anjukeId)) + return detail.findOne(q) + } + + def nameFileter(def region, def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + name = name.replace("(${region})","").replace("(","").replace(")","") + return name; + } + + def getEstatType(type){ + if(type){ + if(type=="公寓住宅"){ + type = "普通住宅" + }else if(type.contains("商住")){ + type = "商住" + }else if(type.contains("别墅")){ + type = "别墅" + }else{ + type = null + } + } + return type + } + + def getBuildType(def buildType){ + def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"] + def list = [] + if(buildType){ + def arrs = buildType.split("\\|") + for(def s : arrs){ + if(types.contains(s)){ + list << s + } + } + if(list){ + return list.join(",") + } + } + return null + } + + def getBuildType2(def buildType){ + def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"] + def list = [] + if(buildType){ + def arrs = buildType.split(",| ") + for(def s : arrs){ + if(types.contains(s)){ + list << s + } + } + if(list){ + return list.join(",") + } + } + return null + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index a6b739898..a97cb7669 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -358,13 +358,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("重庆")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap //cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy index 3ea9f1d4a..e11e01e8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -29,7 +29,7 @@ class JdPaimaiCrawl { task.run(); } def run(){ - def existsIds = findAllIds() + def existsIds = []//findAllIds() println "已存在:"+existsIds.size() System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); @@ -43,78 +43,80 @@ class JdPaimaiCrawl { def getList(driver){ def list = [] - for(def i=1;i<=32;i++){ - if(i==8){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" - }else if(i==5){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" - }else if(i==17){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" - }else if(i==19){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" - }else{ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" - } - } + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268" +// for(def i=1;i<=32;i++){ +// if(i==8){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" +// }else if(i==5){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" +// }else if(i==17){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" +// }else if(i==19){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" +// }else{ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" +// } + +// } def links = [] for(def url : list){ driver.get(url) @@ -430,10 +432,10 @@ class JdPaimaiCrawl { def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc) if(s1) { - def start = s1.indexOf("法院将于") + def start = s1.indexOf("将于") def end = s1.indexOf("止") if(start>-1 && end>-1 && end>start){ - s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止")) } def dateRangs = DataUtil.getDateRange(s1) if(dateRangs && dateRangs.size()==2) { @@ -476,6 +478,11 @@ class JdPaimaiCrawl { } } + + if(!result.startDate){ + def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","") + result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日") + } if(result.finishDate){ result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6f99480ab..cae68c08d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,10 +34,30 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); - task.run() +// task.run() // task.run1() - task.addDealRegion(); - task.standardRegion() +// task.addDealRegion(); +// task.standardRegion() + task.updadteDealPriceField() + } + + def updadteDealPriceField(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def city : citys){ + index ++ + def q = new BasicDBObject() + q.put("city", city) + q.put("hall", "0") + def count = DEAL_PRICE_COLL.count(q) + println "[${index}] "+city +" "+count + if(count && count>0){ + def udata = new BasicDBObject() + udata.put("hall", 0) + DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true); + println "[修改]"+city +" "+ count + } + } } def run1(){ @@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["长春","吉林"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index c7451c5af..6a50bd201 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -30,8 +30,8 @@ class PaimaiTest { def task = new PaimaiTest(); // task.run(); // task.stat() -// task.updatePaimai() - task.updateFieldPaimai() + task.updatePaimai() +// task.updateFieldPaimai() } def updateFieldPaimai(){ @@ -42,8 +42,8 @@ class PaimaiTest { for(def obj : dbcursor) { index++ println index - if(obj.depositPrice){ - obj.put("depositPrice", obj.depositPrice.toInteger()) + if(obj.address && obj.address.contains("可贷款")){ + obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款"))) paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) } } @@ -51,11 +51,22 @@ class PaimaiTest { def updatePaimai(){ def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) def index =0 for(def d : dbcursor){ index++ println index + if(!d.url.contains("jd.com")){ + continue + } + + if(!d.proince.contains("重庆")){ + continue + } + + println d.url + def query = new BasicDBObject() query.put("url", d.url) def dbo = paimai43_COLL.findOne(query); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index 9fbafb2ae..2a231a42c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -42,8 +42,41 @@ class CqfdcMatch { // task.export2() // task.matchLoudong() - task.addLoudong() +// task.addLoudong() // task.addFdcLoudong() + task.updateHouseLoudong() + } + + def updateHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + + def keys = new BasicDBObject() + keys.put("room_num", 1) + keys.put("house_floor", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def index = 0 + for(def d : dbcursor){ + index++ + def room_num = d.room_num + def house_floor = d.house_floor + try{ + if(room_num && house_floor && room_num.isNumber()){ + if(house_floor.toInteger()>0){ + if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){ + def newValue = house_floor +"-"+room_num + println "[${index}] "+room_num +" "+house_floor+" "+newValue + def udata = new BasicDBObject() + udata.put("room_num",newValue) + HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false); + } + } + } + }catch(Exception e){ + println d._id.toString() + } + + } } def export(){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 97af5b6dc..d7222f3d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "58同城"// + def source = "贝壳找房"// def citys = findCrawlCity(source, 202202); for(def city : citys){ - def count1 = findCount(source, city, 202202); - def count2 = findCount(source, city, 202203); + def count1 = findCount(source, city, 202203); + def count2 = findCount(source, city, 202204); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index c676385ed..4784e512e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); -// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] + def citys = ["上海","北京","广州","天津","武汉","南京","青岛","合肥","西安","沈阳","宁波","佛山","潍坊","济南","大连","兰州","石家庄","长春","温州","乌鲁木齐","太原","惠州","嘉兴","南通","南昌","烟台","临沂","呼和浩特","厦门","海口","常州","中山","绍兴","淄博","洛阳","珠海","金华","威海","德阳","唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","衡阳","滨州","张家口","赣州","商丘","安庆","湖州","枣庄","聊城","阜阳","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳","蚌埠","九江","临汾","大同","六安","淮南","宿州","三亚","玉溪","六盘水","揭阳","汕尾","恩施","盘锦","伊犁","玉林","随州","抚州","景德镇","四平","丽水","安顺","毕节","天水","阜新","通化","攀枝花","吕梁","阿克苏","新余","松原","楚雄","鹤壁","忻州","本溪","拉萨","三门峡","黔东南","兴安盟","雅安","仙桃","乌海","贵港","安康","白城","平凉","庆阳","延安","潮州","白银","黔西南","丽江","吴忠","防城港","喀什","哈尔滨","钦州","鸡西","石嘴山","哈密","萍乡","鹰潭","武威","保山","铜仁","普洱","酒泉","宜昌","河池","朔州","百色","鹤岗","张家界","湘西","张掖","定西","昭通","济源","中卫","黑河","克拉玛依","铜川","辽源","云浮","博尔塔拉","固原","白山","广元","贺州","潜江","天门","伊春","石河子","临沧","榆林","临夏","来宾","成都","双鸭山","梧州","嘉峪关","海东","昌吉"] + // for(def city : citys){ // def str = cityList.get(city) // for(def i=0;i<1;i++){ @@ -209,7 +210,7 @@ class Esf58Crawl extends BaseBatch{ cityList.each{city, str-> for(def i=0;i<1;i++){ -// if(citys.contains(city)){ + if(citys.contains(city)){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); if(districts){ def arrs = str.split("\\|") @@ -219,7 +220,7 @@ class Esf58Crawl extends BaseBatch{ addCrawlKeys(city); addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); } -// } + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index fec88b8c5..b620f3098 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"] + def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -26,7 +26,7 @@ class SynchDistrictPrice { def run(){ - def citys = CommonUtils.findAllCity() + def citys = ["重庆"]//CommonUtils.findAllCity() sourceMap.each { k, v -> for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 29c5f5f4b..1a45be219 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -236,10 +236,10 @@ public class OpenApiClient { params.put("houseNumber", houseNum); params.put("unitNumber", unitNum); params.put("token", TOKEN); - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8") println("${id} ${houseNum} 程序调用异常") Thread.sleep(6000) } @@ -266,10 +266,10 @@ public class OpenApiClient { params.put("unitNumber", unitNum); params.put("roomNumber", roomNum); params.put("token", TOKEN); - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") println("${id} ${houseNum} 程序调用异常") Thread.sleep(6000) } @@ -687,7 +687,7 @@ public class OpenApiClient { params.put("region", region) params.put("name", name) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result @@ -697,13 +697,14 @@ public class OpenApiClient { return null } - static JSONObject searchPaimais(String city, String region, String name, limit) { + static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) { Map params = new HashMap(); params.put("token", TOKEN); params.put("city", city) params.put("district", region) params.put("name", name) - params.put("limit", "10") + params.put("startDate", startDate) + params.put("limit", "100") try { String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8") JSONObject result = JSONObject.fromObject(content); @@ -722,7 +723,7 @@ public class OpenApiClient { params.put("startDate", startDate) params.put("limit", limit.toString()) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { @@ -738,7 +739,7 @@ public class OpenApiClient { params.put("startTime", startDate) params.put("pageSize", limit.toString()) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { @@ -755,7 +756,7 @@ public class OpenApiClient { params.put("startDate", startDate) params.put("limit", limit.toString()) try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { From 04e57b12f3c549b2e47c3f422af94f963e6d2246 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 22 May 2022 14:18:17 +0800 Subject: [PATCH 46/83] =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 4978bd962..8d7bf9732 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) From 195916001234f20a5f07f887446b363e49f781fd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 22 May 2022 15:54:50 +0800 Subject: [PATCH 47/83] update --- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- .../script/ccbevaluation/TradeStatLog.groovy | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index e262bd7c5..d1cc4209b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202201 - def endDate = 202203 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3ed13ebfc..3b3e6a7c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-01-01" - def date = "2022-02-01" - def endDate = "2022-03-01" - def newDate = "2022-04-01" + def startDate = "2022-02-01" + def date = "2022-03-01" + def endDate = "2022-04-01" + def newDate = "2022-05-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据") println "表格生成完成" } From 9312b352b0abc51323b6753e77a30ea99a7bd6e4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 22 May 2022 18:49:07 +0800 Subject: [PATCH 48/83] =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 39 +++++++++++++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 060914122..af8723b42 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -38,8 +38,8 @@ class ExportOracleData { exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-03-19") - exportBigdataDeal("2022-03-01") + exportBigdataForsale("2022-04-19") + exportBigdataDeal("2022-04-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 26c6a3e15..af552d3d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -590,8 +590,8 @@ class CommonDatautils { } - public static def findDistrict(String city, String region, String name, String cat){ - + public static def findDistrictCount(String city, String region, String name, String cat){ + DBObject q = new BasicDBObject(); q.put("city", city); if(region){ @@ -602,7 +602,24 @@ class CommonDatautils { q.put("cat", cat) q.put("status", 0) - + def count = getDataStoreConnection().count(q); + return count; + } + + + public static def findDistrict(String city, String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + if(region){ + q.put("region", region); + } + + q.put("name", name); + q.put("cat", cat) + q.put("status", 0) + + DSCursor cursor = getDataStoreConnection().find(q); if(cursor){ for(def obj : cursor){ @@ -613,6 +630,7 @@ class CommonDatautils { } + public static def findDistrictList(String city, String region, String name, String cat){ DBObject q = new BasicDBObject(); @@ -837,12 +855,18 @@ class CommonDatautils { keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) keys.put("sid", 1) - DSCursor cursor = getDataStoreConnection().find(q, keys); - if(cursor){ - for(def obj : cursor){ - return obj; + try{ + DSCursor cursor = getDataStoreConnection().find(q, keys); + if(cursor){ + for(def obj : cursor){ + return obj; + } } + }catch(Exception e){ + println "error_id:"+id + e.printStackTrace() } + return null; } @@ -2398,6 +2422,7 @@ class CommonDatautils { q.put("block", block); } q.put("cat", cat); + q.put("status",0) def cur = getDataStoreConnection().find(q); return cur.count(); } From 13a146e4c6962a6402fbe98db81fe4e46a0b68a2 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 10:30:42 +0800 Subject: [PATCH 49/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=80=E6=88=BF?= =?UTF-8?q?=E4=B8=80=E4=BB=B7=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 96a414024..eec63ef82 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -178,6 +178,7 @@ public class PropertyController { private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; private CommunityPoiServiceImpl communityPoiService; + private EvaluateFactorService evaluateFactorService; private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; @@ -1170,6 +1171,22 @@ public class PropertyController { mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); return mav; } + // 调整面积不同面积价格 + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + evaluation.setAvgPrice(avgPrice); + evaluation.setTotalPrice((int) (avgPrice * evaluation.getArea())); + } Map districtPriceMap = new LinkedHashMap(); districtPriceMap.put("city", district.getCity()); districtPriceMap.put("district", district.getRegion()); @@ -5631,4 +5648,8 @@ public class PropertyController { public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) { this.communityPoiService = communityPoiService; } + + public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) { + this.evaluateFactorService = evaluateFactorService; + } } From edf9694804d7669d685ef0f532149e62a922d545 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 11:08:27 +0800 Subject: [PATCH 50/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BD=AE=E4=BF=A1?= =?UTF-8?q?=E5=BA=A6=E5=90=8C=E6=AD=A5=E8=87=B385?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../OracleDistrict2Mongo85.groovy | 2 +- .../OracleDistrictBasePrice2Mongo85.groovy | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy index 0855b1a8e..93f18dfce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy @@ -30,7 +30,7 @@ import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong /** - * oralce中小区统计数据同步至85中 + * oracle中小区统计数据同步至85中 */ class OracleDistrict2Mongo85 { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy new file mode 100644 index 000000000..64449541a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DistrictCodeUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql + +/** + * oracle中小区置信度数据同步至85中 + */ +class OracleDistrictBasePrice2Mongo85 { + + static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017) + + static main(args) { + def sql = "SELECT cityname, distname, commid, commname, confi_rate FROM community_base_price_gj WHERE YEAR_MONTH='202205'" + def count = 0 + sqlInstance.eachRow(sql, { r -> + def city = r['cityname'] + def region = RegionData.formatRegion(city, r['distname']) + DBObject dbObject = new BasicDBObject("_id", r['commid']) + if (r['confi_rate'] != null) { + def credit = new BigDecimal(r['confi_rate']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble() + DBObject updateObject = new BasicDBObject("city", city).append("region", region) + .append("name", r['commname']) + .append("credit", credit) + .append("c_date", new Date()); + communityBasePriceConn.update(dbObject, updateObject, true, false) + count++; + } else { + print "置信度为空:" + r.toString() + } + if (count % 500 == 0) { + println "已写入" + count + "条小区数据" + } + }); + } +} From a9e7c4fa4b668c1209c24b52809a7c4bedd41db5 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 11:45:50 +0800 Subject: [PATCH 51/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BD=AE=E4=BF=A1?= =?UTF-8?q?=E5=BA=A6=E5=90=8C=E6=AD=A5=E8=87=B385?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy index 64449541a..a824a3d7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy @@ -27,14 +27,14 @@ class OracleDistrictBasePrice2Mongo85 { static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017) static main(args) { - def sql = "SELECT cityname, distname, commid, commname, confi_rate FROM community_base_price_gj WHERE YEAR_MONTH='202205'" + def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'" def count = 0 sqlInstance.eachRow(sql, { r -> def city = r['cityname'] def region = RegionData.formatRegion(city, r['distname']) DBObject dbObject = new BasicDBObject("_id", r['commid']) - if (r['confi_rate'] != null) { - def credit = new BigDecimal(r['confi_rate']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble() + if (r['confidence'] != null) { + def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble() DBObject updateObject = new BasicDBObject("city", city).append("region", region) .append("name", r['commname']) .append("credit", credit) From 70338124281a42f14e4502e85de8b48351bc912a Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 13:20:19 +0800 Subject: [PATCH 52/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=80=E6=88=BF?= =?UTF-8?q?=E4=B8=80=E4=BB=B7=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 115 ++++++++++++------ 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index eec63ef82..69c1eb263 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -397,7 +397,7 @@ public class PropertyController { /** * 利用百度API把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -419,7 +419,7 @@ public class PropertyController { /** * 利用算法模板把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -611,7 +611,7 @@ public class PropertyController { /** * 判断小区是否有经纬度 - * + * * @param district * @return */ @@ -622,7 +622,7 @@ public class PropertyController { /** * 调用匹配接口匹配小区 - * + * * @param city * @param address * @param region @@ -1000,6 +1000,32 @@ public class PropertyController { } } + // 调整面积不同面积价格 + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + + } + if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0 && (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) { Map districtPriceMap = new HashMap(); @@ -1172,20 +1198,29 @@ public class PropertyController { return mav; } // 调整面积不同面积价格 - DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); - int orgAvgPrice = 0; - if (districtStatBo != null) { - if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { - orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); - } else if (districtStatBo.getDistrictStat() != null) { - orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } } - } - if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { - int factorPrice = evaluation.getAvgPrice(); - int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); - evaluation.setAvgPrice(avgPrice); - evaluation.setTotalPrice((int) (avgPrice * evaluation.getArea())); + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + } Map districtPriceMap = new LinkedHashMap(); districtPriceMap.put("city", district.getCity()); @@ -1512,7 +1547,7 @@ public class PropertyController { return mav; } } - + /** * 四川锦程 定制接口 多建筑年代 * @param model @@ -1624,7 +1659,7 @@ public class PropertyController { districtPriceMap.put("buildYear", matcher3.group()); } } - + // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -2137,7 +2172,7 @@ public class PropertyController { /** * 小区租金查询接口 - * + * * @param model * @return */ @@ -2616,7 +2651,7 @@ public class PropertyController { /** * 批量获取小区的均价,均价走势数据 - * + * * @param request * @return */ @@ -2758,7 +2793,7 @@ public class PropertyController { /** * 批量获取小区的配套数据,最多12个小区 - * + * * @param request * @return */ @@ -2903,7 +2938,7 @@ public class PropertyController { /** * 获取JSON参数中的num参数值 - * + * * @param jo * @return */ @@ -2928,7 +2963,7 @@ public class PropertyController { /** * 获取JSON参数中的radius值 - * + * * @param jo * @return */ @@ -3736,7 +3771,7 @@ public class PropertyController { /** * 解析出关键字中的小区名 - * + * * @param keyword * @return */ @@ -3853,7 +3888,7 @@ public class PropertyController { /** * 小区统计分析接口 - * + * * @param model * @return */ @@ -3938,7 +3973,7 @@ public class PropertyController { /** * 小区交通接口 - * + * * @param model * @return */ @@ -3996,7 +4031,7 @@ public class PropertyController { /** * 小区周边接口 - * + * * @param model * @return */ @@ -4055,7 +4090,7 @@ public class PropertyController { /** * 相同小区相似户型房源挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4095,7 +4130,7 @@ public class PropertyController { /** * 周边小区相似户型挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4146,7 +4181,7 @@ public class PropertyController { /** * 小区周边一公里均价 - * + * * @author wenyan * @param model * @return @@ -4169,7 +4204,7 @@ public class PropertyController { /** * 相同小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4212,7 +4247,7 @@ public class PropertyController { /** * 周边小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4275,7 +4310,7 @@ public class PropertyController { /** * 室号查询接口 - * + * * @param model * @return */ @@ -4311,7 +4346,7 @@ public class PropertyController { /** * 楼栋楼层信息查询接口 - * + * * @param model * @return */ @@ -4336,7 +4371,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4381,7 +4416,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4834,7 +4869,7 @@ public class PropertyController { /** * 房产交易记录查询 - * + * * @param model * @return */ @@ -4894,7 +4929,7 @@ public class PropertyController { /** * 房产拍卖记录 - * + * * @param model * @return */ @@ -4955,7 +4990,7 @@ public class PropertyController { /** * 房产近期评估次数 - * + * * @param model * @return */ From 1648838af9abb97fb78a8402e7f8fd202e895c16 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 23 May 2022 13:30:18 +0800 Subject: [PATCH 53/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BD=AE=E4=BF=A1?= =?UTF-8?q?=E5=BA=A6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mongo/open/dao/CommunityBasePriceDao.java | 9 +++ .../dao/impl/CommunityBasePriceDaoImpl.java | 10 +++ .../mongo/open/model/CommunityBasePrice.java | 74 +++++++++++++++++++ .../open/controllers/PropertyController.java | 21 ++++-- .../open/CommunityBasePriceService.java | 9 +++ .../impl/CommunityBasePriceServiceImpl.java | 22 ++++++ 6 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java new file mode 100644 index 000000000..8275f68f8 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java new file mode 100644 index 000000000..c89477b09 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public class CommunityBasePriceDaoImpl extends MongoDaoImpl implements CommunityBasePriceDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java new file mode 100644 index 000000000..9321bd542 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java @@ -0,0 +1,74 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +/** + * 小区建行置信度信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "business.community_base_price_gj") +public class CommunityBasePrice implements MongoEntity { + + private static final long serialVersionUID = -8010010773207622878L; + + private String id; + private String city; + private String region; + private String name; + private Double credit; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "credit") + public Double getCredit() { + return credit; + } + + public void setCredit(Double credit) { + this.credit = credit; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 69c1eb263..0a7df7866 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -31,6 +31,7 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; import com.fangjia.dal.mongo.model.*; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; import com.fangjia.dal.mongo.open.model.CommunityPoi; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; @@ -177,7 +178,8 @@ public class PropertyController { private SeawaterDao seawaterDao; private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; - private CommunityPoiServiceImpl communityPoiService; + private CommunityPoiService communityPoiService; + private CommunityBasePriceService communityBasePriceService; private EvaluateFactorService evaluateFactorService; private FileDao fileDao = FileDao.getInstance(); @@ -1235,6 +1237,13 @@ public class PropertyController { // } districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); + // 新增置信度 + CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId()); + if (communityBasePrice != null) { + districtPriceMap.put("credit", communityBasePrice.getCredit()); + } else { + districtPriceMap.put("credit", evaluation.getCredit()); + } List names = new ArrayList(); names.add(district.getName()); if(district.getAlias() != null){ @@ -5676,15 +5685,15 @@ public class PropertyController { this.rentDealPriceService = rentDealPriceService; } - public CommunityPoiServiceImpl getCommunityPoiService() { - return communityPoiService; - } - - public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) { + public void setCommunityPoiService(CommunityPoiService communityPoiService) { this.communityPoiService = communityPoiService; } public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) { this.evaluateFactorService = evaluateFactorService; } + + public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { + this.communityBasePriceService = communityBasePriceService; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java new file mode 100644 index 000000000..bc8c159de --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java @@ -0,0 +1,9 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceService { + + CommunityBasePrice getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java new file mode 100644 index 000000000..049e5ae16 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java @@ -0,0 +1,22 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityBasePriceService; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityBasePriceServiceImpl implements CommunityBasePriceService { + + private CommunityBasePriceDao communityBasePriceDao; + + @Override + public CommunityBasePrice getById(String id) { + return communityBasePriceDao.findById(id); + } + + public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) { + this.communityBasePriceDao = communityBasePriceDao; + } +} From a8cb3041a5b9cda7baa1c0b11485cd27a4355b49 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 24 May 2022 13:28:20 +0800 Subject: [PATCH 54/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E5=9D=87=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 0a7df7866..33ab1b0f7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1280,6 +1280,18 @@ public class PropertyController { houseDtos.add(houseDto); } districtPriceMap.put("houses", houseDtos); + // 新增第三方均价 + try { + double dCredit = 0.61; + Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); + if (null != rMap) { + if (rMap.containsKey("other_price")) { + districtPriceMap.put("other_price", rMap.get("other_price")); + } + } + } catch (Exception ex) { + + } }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); From 089971247554197ae89367d684f4061d58d6fe46 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 24 May 2022 13:53:42 +0800 Subject: [PATCH 55/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E5=9D=87=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 33ab1b0f7..fcf240ddc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -396,6 +396,27 @@ public class PropertyController { put("5000", "five");// 五线 } }; + /** + * 第三方均价映射 + */ + private static final Map OTHER_PRICE_MAP = new HashMap() { + { + put("zhuge_price", "诸葛找房"); + put("anjuke_price", "安居客"); + put("fang_price", "房天下"); + put("lianjia_price", "链家"); + put("cityhouse_price", "禧泰数据"); + put("maitian_price", "麦田"); + put("ke_price", "贝壳"); + put("taofang_price", "淘房365"); + put("zhongyuan_price", "中原"); + put("tbs_price", "兔博士"); + put("qfang_price", "Q房"); + put("fangtan_price", "房探"); + put("fangduoduo_price", "房多多"); + put("wiwj_price", "我爱我家"); + } + }; /** * 利用百度API把GPS转换成Google经纬度 @@ -1255,6 +1276,8 @@ public class PropertyController { } Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8); Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); + Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9); + Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1); List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); if(houses.size() > 3){ districtPriceMap.put("calcType", 1); @@ -1286,7 +1309,18 @@ public class PropertyController { Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); if (null != rMap) { if (rMap.containsKey("other_price")) { - districtPriceMap.put("other_price", rMap.get("other_price")); + Map otherPriceMap = (Map) rMap.get("other_price"); + Map priceResultMap = new HashMap(); + for (String key : otherPriceMap.keySet()) { + Integer price = otherPriceMap.get(key); + // 剔除偏差大于10的 + if (min10Price <= price && max10Price >= price) { + if (OTHER_PRICE_MAP.containsKey(key)) { + priceResultMap.put(OTHER_PRICE_MAP.get(key), price); + } + } + } + districtPriceMap.put("otherPrice", priceResultMap); } } } catch (Exception ex) { From 78b9a5a52220fe452d4e788a4db1718632b264c4 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 24 May 2022 14:30:02 +0800 Subject: [PATCH 56/83] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index fcf240ddc..89d72a2cc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1278,6 +1278,29 @@ public class PropertyController { Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9); Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1); + // 新增第三方均价 + try { + double dCredit = 0.61; + Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); + if (null != rMap) { + if (rMap.containsKey("other_price")) { + Map otherPriceMap = (Map) rMap.get("other_price"); + Map priceResultMap = new HashMap(); + for (String key : otherPriceMap.keySet()) { + Integer price = otherPriceMap.get(key); + // 剔除偏差大于10的 + if (min10Price <= price && max10Price >= price) { + if (OTHER_PRICE_MAP.containsKey(key)) { + priceResultMap.put(OTHER_PRICE_MAP.get(key), price); + } + } + } + districtPriceMap.put("otherPrice", priceResultMap); + } + } + } catch (Exception ex) { + + } List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); if(houses.size() > 3){ districtPriceMap.put("calcType", 1); @@ -1303,29 +1326,6 @@ public class PropertyController { houseDtos.add(houseDto); } districtPriceMap.put("houses", houseDtos); - // 新增第三方均价 - try { - double dCredit = 0.61; - Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); - if (null != rMap) { - if (rMap.containsKey("other_price")) { - Map otherPriceMap = (Map) rMap.get("other_price"); - Map priceResultMap = new HashMap(); - for (String key : otherPriceMap.keySet()) { - Integer price = otherPriceMap.get(key); - // 剔除偏差大于10的 - if (min10Price <= price && max10Price >= price) { - if (OTHER_PRICE_MAP.containsKey(key)) { - priceResultMap.put(OTHER_PRICE_MAP.get(key), price); - } - } - } - districtPriceMap.put("otherPrice", priceResultMap); - } - } - } catch (Exception ex) { - - } }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); From 420f0d046e41a9d1f45054ac942913e820e1bfee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 25 May 2022 10:04:42 +0800 Subject: [PATCH 57/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/processor/CityProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 130b7eea9..12adc0741 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -198,8 +198,6 @@ public class CityProcessor extends AbstractAddressProcessor { index += city.length(); if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){ weight.score += 2; - }else{ - weight.score += 1; } } Map> regionMap = RegionData.getAllRegions(city); From 4c7041b8b82e9ad2306a8ae23b253c713a780129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 25 May 2022 12:01:23 +0800 Subject: [PATCH 58/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/processor/CityProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 12adc0741..fce876ec2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -221,7 +221,7 @@ public class CityProcessor extends AbstractAddressProcessor { List alias = regionMap.get(region); if(alias != null){ for(String alia : alias){ - if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1 + if(StringUtils.isNotBlank(alia) && !alia.equals(region) && alia.length() > 2 && address.indexOf(alia, index) > -1 && (alia.endsWith("市")|(alia.endsWith("县")))){ if(city.equals(addr.getCity())){ weight.score += 10; From 41b0b31b7587bdd87d8bfe60e2ebc4d29c39837d Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 14:05:55 +0800 Subject: [PATCH 59/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E5=86=99=E5=85=A5=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../poi/SeaweedAroundPoiSingal.groovy | 106 ++++++++++-------- 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy index 1a620af34..08b77989b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel import org.apache.commons.collections.CollectionUtils +import org.bson.Document +import org.bson.conversions.Bson import org.bson.types.ObjectId import java.util.concurrent.ExecutorService @@ -27,74 +33,82 @@ import java.util.concurrent.Executors */ class SeaweedAroundPoiSingal { - static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static void main(String[] args) { - def types = listAllType() - //def types = "0601" - DBObject query = new BasicDBObject() - query.put("status", 1) - conn136.find(query).batchSize(30).each {objTemp -> - final def obj = objTemp - def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) - } - println("已处理小区:" + obj) - } + getNewSeaweed() +// def types = listAllType() +// //def types = "0601" +// DBObject query = new BasicDBObject() +// query.put("status", 1) +// conn136.find(query).batchSize(30).each {objTemp -> +// final def obj = objTemp +// def city = obj.city +// def lng = obj.lng +// def lat = obj.lat +// List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") +// if (CollectionUtils.isNotEmpty(pois)) { +// String jsonStr = JSONObject.toJSONString(pois) +// HttpUtils.postForJson(URL, jsonStr) +// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) +// } else { +// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) +// } +// println("已处理小区:" + obj) +// } } static getNewSeaweed() { DBObject query = new BasicDBObject() query.put("cat", "district") query.put("visible", 0) - //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) DBObject filter = new BasicDBObject() filter.put("_id", 1) filter.put("city", 1) filter.put("region", 1) filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) filter.put("lng", 1) filter.put("lat", 1) - List list = [] + List> writeModelList = new ArrayList<>() DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> - if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { - // 已存在小区,无需同步 - } else { - DBObject newObj = new BasicDBObject() - newObj.put("_id", obj._id) - newObj.put("city", obj.city) - newObj.put("region", obj.region) - newObj.put("name", obj.name) - try { - newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) - newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) - } catch (Exception ex) { - print("经纬度为空:" + obj._id + " " + obj) - ex.printStackTrace() - } - newObj.put("status", 1) - newObj.put("flag", "未获取周边POI") + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) - list << newObj - if (list.size() == 5000) { - conn136.insert(list) - list.clear() - } + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() } } - conn136.insert(list) - list.clear() + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } } static def getByIdAndInsert(String id) { From 4f2e84a5d96045f408e2d16f11ccf1f96c574868 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 14:40:20 +0800 Subject: [PATCH 60/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E5=86=99=E5=85=A5=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy index 08b77989b..20def0a9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -76,6 +76,7 @@ class SeaweedAroundPoiSingal { filter.put("lat", 1) List> writeModelList = new ArrayList<>() + def count = 0 DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> Document updateBson = new Document() Document update = new Document() @@ -102,6 +103,8 @@ class SeaweedAroundPoiSingal { writeModelList.add(updateOneModel) if (writeModelList.size() == 5000) { conn136.bulkWrite(writeModelList) + count += 5000 + println(count) writeModelList.clear() } } From d57955ef1ee42fce708bae97e15fd33edd2860ae Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 16:43:10 +0800 Subject: [PATCH 61/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E9=AB=98=E5=BE=B7=E6=90=9C=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedPoiDetail.groovy | 157 ++++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 31 ++++ .../script/statistics/poi/AmapPoiDetail.java | 71 ++++++++ 3 files changed, 259 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy new file mode 100644 index 000000000..feffbc12b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -0,0 +1,157 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.bson.conversions.Bson +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区POI信息精确查找 + * + * @author shenjian + * @since 2022/5/25 + */ +class SeaweedPoiDetail { + + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + static final ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("status", 1) + query.put("city", "东莞") + conn136.find(query).batchSize(1000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def region = obj.region + def name = obj.name + def alias = obj.alias + def address = obj.address + def addr_alias = obj.addr_alias + StringJoiner stringJoiner = new StringJoiner("|") + // 主小区名搜索 + stringJoiner.add(name) + // 根据别名搜 + if (alias != null && CollectionUtils.isNotEmpty(alias)) { + for (String str : alias) { + stringJoiner.add(str) + } + } + // 根据路名搜 + if (StringUtils.isNotBlank(address)) { + stringJoiner.add(address) + } + // 根据路别名搜 + if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) { + for (String str : addr_alias) { + stringJoiner.add(str) + } + } + + String keywords = stringJoiner.toString() + List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes) + + List detailList = new ArrayList<>() + for (AmapPoi amapPoi : pois) { + List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId()); + if (CollectionUtils.isNotEmpty(amapPoiList)) { + detailList.addAll(amapPoiList) + } + } + if (CollectionUtils.isNotEmpty(detailList)) { + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到"))) + } else { + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区"))) + } + println("已处理小区:" + obj) + } + }) + } + executorService.shutdown() + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List> writeModelList = new ArrayList<>() + def count = 0 + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) + + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + count += 5000 + println(count) + writeModelList.clear() + } + } + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 49c7c18de..aae3318c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import org.apache.commons.lang.StringUtils +import org.openqa.selenium.json.Json import java.util.concurrent.atomic.AtomicInteger @@ -232,6 +234,35 @@ class GaoDeUtils { return Collections.emptyList() } + /** + * 获取详情 + */ + static List getAmapDetailById(id){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.id = id + params.output = 'json' + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return Collections.emptyList() + } + /** * 查找关键字经纬度 */ diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java new file mode 100644 index 000000000..8192b359c --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2022/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoiDetail { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From 3b5b567a3c6c469b4c29d1e9773e3798ed9ca72c Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 17:49:59 +0800 Subject: [PATCH 62/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=90=9C=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy index feffbc12b..b40ae6cbf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -78,6 +78,9 @@ class SeaweedPoiDetail { String keywords = stringJoiner.toString() List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes) + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, name, region, typecodes) + } List detailList = new ArrayList<>() for (AmapPoi amapPoi : pois) { From 3b97cb5c8c80389f0566eabddb5adbb6bc33468d Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 19:31:45 +0800 Subject: [PATCH 63/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E8=B6=85=E9=99=90=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedPoiDetail.groovy | 1 - .../temp/shenjian/util/GaoDeUtils.groovy | 52 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy index b40ae6cbf..dbdd337aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -44,7 +44,6 @@ class SeaweedPoiDetail { static void main(String[] args) { DBObject query = new BasicDBObject() query.put("status", 1) - query.put("city", "东莞") conn136.find(query).batchSize(1000).each {objTemp -> final def obj = objTemp executorService.submit(new Runnable() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index aae3318c9..33ae920f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -90,12 +90,25 @@ class GaoDeUtils { if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() result.addAll(DataUtil.toMongoDbObj(json.pois)) - } else if (json && json.infocode && json.infocode == 10003) { - throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") - } else if (json && json.infocode && json.infocode == 10004) { + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) } else { break } @@ -224,6 +237,17 @@ class GaoDeUtils { } catch(Exception ex) { ex.printStackTrace() } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) @@ -253,6 +277,17 @@ class GaoDeUtils { } catch(Exception ex) { ex.printStackTrace() } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) @@ -398,7 +433,16 @@ class GaoDeUtils { return ["lng": arrs[0], "lat": arrs[1]] } } else if (json && json.infocode && json.infocode == "10044") { - System.exit(1) + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From 299e9e81a013d9cc698afe9fd6546c80589f0b21 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 19:37:59 +0800 Subject: [PATCH 64/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E8=B6=85=E9=99=90=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 33ae920f4..ca4a2ab38 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -94,7 +94,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -161,7 +161,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -241,7 +241,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -281,7 +281,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -336,7 +336,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -392,7 +392,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -436,7 +436,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) From 082f7fff2080af428ca5b91ec2c472ff9dc32575 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 19:42:57 +0800 Subject: [PATCH 65/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=90=9C=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedPoiDetail.groovy | 15 +++---- .../temp/shenjian/util/CommonDtoUtils.groovy | 40 +++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy index dbdd337aa..3ae943007 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils @@ -81,13 +82,13 @@ class SeaweedPoiDetail { pois = GaoDeUtils.searchPoi(city, name, region, typecodes) } - List detailList = new ArrayList<>() - for (AmapPoi amapPoi : pois) { - List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId()); - if (CollectionUtils.isNotEmpty(amapPoiList)) { - detailList.addAll(amapPoiList) - } - } + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); +// for (AmapPoi amapPoi : pois) { +// List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId()); +// if (CollectionUtils.isNotEmpty(amapPoiList)) { +// detailList.addAll(amapPoiList) +// } +// } if (CollectionUtils.isNotEmpty(detailList)) { String jsonStr = JSONObject.toJSONString(detailList) HttpUtils.postForJson(URL, jsonStr) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy new file mode 100644 index 000000000..89dda4684 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.temp.shenjian.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * 通用实体与DTO互相转换工具类 + * + * @author shenjian + * @date 2021/06/16 + */ +public class CommonDtoUtils { + + public static T transformV(Class targetClass, Object source) { + if (source == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(source); + return JSONObject.parseObject(jsonSource, targetClass); + } catch (Exception var3) { + throw var3; + } + } + } + + public static List transformListV(Class targetClass, List listSource) { + if (listSource == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(listSource); + return JSONArray.parseArray(jsonSource, targetClass); + } catch (Exception var3) { + throw var3; + } + } + } +} \ No newline at end of file From cfa7fa9e7e726699afa94f8fdb39634878d1c5a6 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 25 May 2022 19:46:27 +0800 Subject: [PATCH 66/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=90=9C=E7=B4=A2key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index ca4a2ab38..b345ef414 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger class GaoDeUtils { // 新获取key - def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"] + def static GAO_DE_AK = ["4ccfe3c898a9489884bd2bcb1ef6dcde", "4e9592208336433ee1b7aa4d1d434418"] def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ From 8c8db4ad60348d2354977260e36f96d8d8f22901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 25 May 2022 19:51:25 +0800 Subject: [PATCH 67/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/processor/KeywordProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java index 547d8d2cb..b982d0b4f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java @@ -22,7 +22,7 @@ public class KeywordProcessor extends AbstractAddressProcessor { static final List KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号")); static final List KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区"); static final List NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$")); - static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属院")); + static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属[区院]"), Pattern.compile("宿舍"), Pattern.compile("生活区"), Pattern.compile("公寓")); static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$"); From e59a83be25d12fa229f3f7a9280e14eeb4147611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 25 May 2022 21:01:39 +0800 Subject: [PATCH 68/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java index 3965c9f00..c38f74a88 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java @@ -240,7 +240,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { } private boolean isAvailablePoi(AddressMatchMetaData data){ - if(data.getType() != null && data.getType().contains("路")){ + if(data.getType() != null && (data.getType().contains("路") || data.getType().contains("街道"))){ return false; } return true; From df815639794d2e2781c534fd73a5a9a58d74e213 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 26 May 2022 13:38:48 +0800 Subject: [PATCH 69/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E4=B8=8D=E5=8F=AF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index b345ef414..9853b7e79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -175,7 +175,9 @@ class GaoDeUtils { break; } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - System.exit(1) + //System.exit(1) + Thread.sleep(60000) + break; } else { break } @@ -253,7 +255,8 @@ class GaoDeUtils { Thread.sleep(60000) } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - System.exit(1) + //System.exit(1) + Thread.sleep(60000) } return Collections.emptyList() } @@ -293,7 +296,8 @@ class GaoDeUtils { Thread.sleep(60000) } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - System.exit(1) + //System.exit(1) + Thread.sleep(60000) } return Collections.emptyList() } @@ -350,7 +354,9 @@ class GaoDeUtils { break; } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - System.exit(1) + //System.exit(1) + Thread.sleep(60000) + break; } } return result From b8412c64bd155d130048d83a93c5d9cd13db5c3c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 26 May 2022 15:37:40 +0800 Subject: [PATCH 70/83] update --- .../open/controllers/AuctionController.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index f2a10c491..f24e78f40 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -208,7 +208,7 @@ public class AuctionController { if(StringUtils.isNotBlank(model.getStartDate())){ startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); if(startDate != null){ - params.add(new QueryParam("startDate", startDate)); + params.add(new QueryParam("startDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); } } @@ -219,9 +219,15 @@ public class AuctionController { for(PaiMai paiMai : paimais){ Map map = standard(paiMai.getCity(), paiMai.getAddress()); if(map!=null){ - paiMai.setHouseNum((String) map.get("buildingNumber")); - paiMai.setUnitNum((String) map.get("unitNumber")); - paiMai.setRoomNum((String) map.get("roomNumber")); + if (map.containsKey("buildingNumber")) { + paiMai.setHouseNum((String) map.get("buildingNumber")); + } + if (map.containsKey("unitNumber")) { + paiMai.setUnitNum((String) map.get("unitNumber")); + } + if (map.containsKey("roomNumber")) { + paiMai.setRoomNum((String) map.get("roomNumber")); + } } } } From 2dec82149d0c13acc4f36c831790e73fbfff04b3 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 26 May 2022 15:38:17 +0800 Subject: [PATCH 71/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 9853b7e79..ea456aa50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -348,14 +348,12 @@ class GaoDeUtils { } break; } - } else if (json && json.infocode && json.infocode == "10004") { + } else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; - } else if (json && json.infocode && json.infocode == "10017") { - // 所请求的资源不可用 - //System.exit(1) - Thread.sleep(60000) + } else if (json && json.infocode && (json.infocode == "20000")) { + // 无效参数 break; } } From 96b0d198606ec0669e095b53bbc35a1dcbd49d70 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 26 May 2022 17:31:59 +0800 Subject: [PATCH 72/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AuctionController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index f24e78f40..b515c4593 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -31,7 +31,8 @@ public class AuctionController { private PaiMaiService paiMaiService; private DistrictNewPriceService districtNewPriceService; - private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType","type","count"}; + private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", + "matchEstateType","type","count", "tradeAvgPrice", "evaluationPrice"}; public Object housesAction(AuctionModel model){ RestfulModelAndView mav = new RestfulModelAndView(); From e6a0da2452e2774f83fb15570e0f3813b149e38e Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 27 May 2022 09:05:50 +0800 Subject: [PATCH 73/83] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=90=9C=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy index 3ae943007..ea6d9cc67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -81,6 +81,9 @@ class SeaweedPoiDetail { if (CollectionUtils.isEmpty(pois)) { pois = GaoDeUtils.searchPoi(city, name, region, typecodes) } + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, address, region, typecodes) + } List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); // for (AmapPoi amapPoi : pois) { From 24609324fbb65083c943f131e80759e76d5ab5a3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 27 May 2022 15:15:33 +0800 Subject: [PATCH 74/83] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 11 +- .../DealAndThirdPriceDataProcess.groovy | 11 +- .../script/ccbevaluation/LoadGuide.groovy | 52 +- .../ccbevaluation/data/CcbDistrictToDB.groovy | 128 ++--- .../ccbevaluation/data/DistrictMarge.groovy | 79 ++- .../ccbevaluation/data/MatchMapData.groovy | 29 +- .../ccbevaluation/data/OracleHouse.groovy | 108 ++++ .../chongqing/MatchEvaluateRegionTest.groovy | 399 +++++++++++++- .../fangjia/script/chongqing/PocStat.groovy | 498 +++++++++++++++++- .../fangjia/script/chongqing/PocTest.groovy | 101 +++- .../script/commData/CommonDatautils.groovy | 20 +- .../fuzhou/DistrictPriceExportBatch.groovy | 91 +++- .../script/fuzhou/FzDataResultStat.groovy | 80 ++- .../com/fangjia/script/fuzhou/Test1.groovy | 4 +- .../script/fuzhou/TradeExportBatch.groovy | 27 +- .../script/gj/crawl/AnjukeAppCrawl.groovy | 79 ++- .../fangjia/script/gj/crawl/AnjukeToDB.groovy | 23 +- .../script/gj/district/NameProcess.groovy | 3 +- .../script/gj/level/DistrictIndexBatch.groovy | 5 +- .../script/gj/meituan/MatchTest.groovy | 50 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 22 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 14 +- .../script/jiangnanblank/CZDataBatch.groovy | 48 +- .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../script/thirdPriceCrawl/DataStat.groovy | 2 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 54 +- .../fangjia/script/util/OpenApiClient.groovy | 36 +- 27 files changed, 1736 insertions(+), 240 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 8d7bf9732..0bc461cc5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//导历史房源需要,以后不需要 // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); @@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{ def map = houses[k]; DBObject seawater = new BasicDBObject(map) + if(estate_type2 && estate_type2=="普通住宅"){ + if(seawater.estate_type&& seawater.estate_type.contains("商住")){ + println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type + continue; + } + } + if(!seawater.total_price){ continue; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index d7840e5bb..c8697c956 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -15,8 +15,8 @@ class DealPriceDataProcess { - def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -373,16 +373,17 @@ class DealPriceDataProcess { def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202203 + def month = 202204 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202204) + def rets = findPeGteThirdParty(city, 202205) def total = 0 + println city +" "+rets.size() for(def m : rets){ def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } - println city +" "+rets.size() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index 42ad78607..b0476927d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -1,21 +1,29 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.fangjia.core.RegionData import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import groovy.sql.Sql +import org.bson.types.ObjectId class LoadGuide { public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { def task = new LoadGuide() task.run() } def run(){ + addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx") // match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") @@ -26,9 +34,51 @@ class LoadGuide { // savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") - savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") } + + def addPersonGuided(filename){ + ExcelUtils.modify(filename, { d -> + def city = d['B'] + if(city){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + createSql( d, cityCodeDatas, distCodeMap) + } + + }) + } + + def createSql(d, cityCodeDatas, distCodeMap){ + def id = new ObjectId() + def commid = d['A'] + def cityId = cityCodeDatas.get(d['B']) + def sampleRegion = d['C'].replace("街道","").replace("镇","") + def distRegion = distCodeMap.get(sampleRegion) + def cityName = d['B'] + def distName = d['C'] + def commName = d['D'] + def price = d['E'].toDouble().toInteger() + + def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district") + if(dist){ + def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + + + def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " + + "('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')" + println sql + sqlInstance.executeUpdate(sql) + }else{ + println " 没查到 "+commName + } + + + } + def match(def filename){ def city = "西安" ExcelUtils.modify(filename,{d-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 2db54e10a..98d755434 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -7,6 +7,7 @@ import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.NameProcess import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.CommonUtils @@ -26,9 +27,9 @@ class CcbDistrictToDB { // task.run() // task.run1() // task.meituanRun() -// task.meituanSqrentRun() + task.meituanSqrentRun() // task.updateDistrict() - task.updateDongguangRegion() +// task.updateDongguangRegion() } def updateDongguangRegion(){ @@ -80,13 +81,17 @@ class CcbDistrictToDB { } def meituanSqrentRun(){ - ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def index = 0 + ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d -> + index++ + println index+" "+d['H'] def city = d['F'] def region = d['G'] def newRegion = RegionData.formatRegion(city, region) if(newRegion) { - def name = d['H'] + def name = NameProcess.nameStandard(city, newRegion, d['H']) name = nameFileter(name) + if(name){ def estateType2 = "普通住宅" if(name.contains("别墅")){ @@ -103,7 +108,6 @@ class CcbDistrictToDB { } } } - def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0) } @@ -115,97 +119,100 @@ class CcbDistrictToDB { def index = 0//美团需入库小区合并后.xlsx def keys = new CopyOnWriteArrayList() - ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d -> + index++ + println index+" "+d['H'] def city = d['F'] def region = d['G'] def newRegion = RegionData.formatRegion(city, region) - if(newRegion){ - def name = d['H'] + if(newRegion) { + def name = NameProcess.nameStandard(city, newRegion, d['H']) name = nameFileter(name) + if (name) { def formatAddress = null - if(d['E']){ - formatAddress = AddressUtil.formatAddress(city, d['E']) + if (d['I']) { + formatAddress = AddressUtil.formatAddress(city, d['I']) } - if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ - def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) - def map = findStandardObject2(city,region, name, d['E'], rets); - if(map && map.address){ - formatAddress = AddressUtil.formatAddress(city, map.address) - if(!formatAddress){ - if(map.address.contains("交叉口")){ - formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) - }else if(map.address.contains("交叉")){ - formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) - } - } - } - } +// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ +// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) +// def map = findStandardObject2(city,region, name, d['E'], rets); +// if(map && map.address){ +// formatAddress = AddressUtil.formatAddress(city, map.address) +// if(!formatAddress){ +// if(map.address.contains("交叉口")){ +// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) +// }else if(map.address.contains("交叉")){ +// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) +// } +// } +// } +// } Long glat = null Long glng = null Long blat = null Long blng = null - if(d['K']){ - glat = d['L'].toString().toDouble() * 10000000000L - glng = d['K'].toString().toDouble() * 10000000000L - def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + if (d['L']) { + glat = d['M'].toString().toDouble() * 10000000000L + glng = d['L'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) blng = baidu.x * 10000000000L blat = baidu.y * 10000000000L } def estateType2 = "普通住宅" - if(name.contains("别墅")){ + if (name.contains("别墅")) { estateType2 = "别墅" - }else if(name.contains("商住")){ + } else if (name.contains("商住")) { estateType2 = "商住" - }else{ - def type = d['I'] - if(type){ - if(type.contains("别墅")){ + } else { + def type = d['J'] + if (type) { + if (type.contains("别墅")) { estateType2 = "别墅" - }else if(type.contains("商住")){ + } else if (type.contains("商住")) { estateType2 = "商住" } } } - def key = city+"##"+newRegion+"##"+name - if(name && !keys.contains(key)){ + def key = city + "##" + newRegion + "##" + name + if (name && !keys.contains(key)) { def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") - if(!dist){ + if (!dist) { keys << key def data = new BasicDBObject(); - data.put("name_cat",0) - data.put("status",0) - data.put("cat","district") + data.put("name_cat", 12) + data.put("status", 0) + data.put("cat", "district") data.put("c_date", new Date()) data.put("m_date", new Date()) data.put("city", city) - data.put("region",newRegion) + data.put("region", newRegion) data.put("name", name) - data.put("estate_type2",estateType2) + data.put("estate_type2", estateType2) data.put("address", formatAddress) - if(glat){ - data.put("lat",glat) - data.put("lng",glng) - data.put("lat2",blat) - data.put("lng2",blng) + if (glat) { + data.put("lat", glat) + data.put("lng", glng) + data.put("lat2", blat) + data.put("lng2", blng) } def markMap = [:] markMap.put("data_source", "机器入库") - data.put("mark_map",markMap) + data.put("mark_map", markMap) index++ DbUtils.seaweed.save(data); - println index+" "+data + println index + " " + data def stop = false - while (!stop){ - if(data._id.toString()){ + while (!stop) { + if (data._id.toString()) { addPrice(data) stop = true; } @@ -213,19 +220,17 @@ class CcbDistrictToDB { } } + } }else{ println city +" "+region +" 区域无法标准化" } },0); - - - } def addPrice(dist){ - def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") - def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24") def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) for(def weekend : weekends){ @@ -435,9 +440,12 @@ class CcbDistrictToDB { } def nameFileter(def name){ - name = name.replace("·","").replace(".","").replace("|","") - name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") - name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + } + return name; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy index a1e095a7f..afa2233b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -1,10 +1,12 @@ package com.fangjia.script.ccbevaluation.data +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import com.mongodb.DBObject import org.apache.poi.ss.util.RegionUtil import org.bson.types.ObjectId import com.fangjia.core.util.BaiDuApiUtils @@ -13,7 +15,82 @@ class DistrictMarge { static main(args) { def task = new DistrictMarge() //task.run() - task.repeatAddress() +// task.repeatAddress() +// task.removeDist() + task.updateAddress() +// task.addMappingAlias() + } + def updateAddress(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d -> + if (d['A']) { + def region = d['B'].trim() + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + DBObject udata = new BasicDBObject() + udata.put("address", d['D']) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + } + }else{ + println d['A'] + " 格式化失败 "+d['B'] + } + } + }); + } + + def removeDist(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + DBObject udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + } + }else{ + println d['A'] + " 格式化失败 "+d['B'] + } + } + }) + } + + def addMappingAlias(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d -> + index ++ + if (d['A']) { + def region = d['B'].trim() + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + def mapping_alias = dist.mapping_alias + if(!dist.mapping_alias){ + mapping_alias = [] + } + def alias = d['D'] + def arrs = alias.split("、") + for(def s : arrs){ + if(!mapping_alias.contains(s)){ + mapping_alias << s + } + } + DBObject udata = new BasicDBObject() + udata.put("mapping_alias", mapping_alias) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + } + }else{ + println d['A'] + " 格式化失败 "+d['B'] + } + } + }) } def repeatAddress(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 2f5d02c2f..a62cf530f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -36,10 +36,37 @@ class MatchMapData { // task.run3() // task.exportHouse() // task.addDistrictLevel() - task.run4() +// task.run4() + task.run5() } + def run5(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\数据核实.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if(dist){ + if(dist.mapping_alias){ + d['D'] = dist.mapping_alias.join(",") + } + d['E'] = dist.address + } + + def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district") + if(dist1){ + if(dist1.mapping_alias){ + d['H'] = dist1.mapping_alias.join(",") + } + d['I'] = dist1.address + } + } + } + }); + } + def run4() { ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d -> def regionCode = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy new file mode 100644 index 000000000..ebb9c165b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy @@ -0,0 +1,108 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import groovy.sql.Sql +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class OracleHouse { + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + + public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + public static EsDataSource dataSource = new EsDataSource() + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new OracleHouse() + task.run() + } + + def run(){ + def rets = findAllHouse("2022-04-19") + def totalCount = new AtomicLong() + println rets.size() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i + index = index + 1 + println index + def data = [] + data << r.FORSALEID + data << r.COMMID + allRows << data + }); + return allRows + } + + def removeHouse(houseid){ + def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' "; + sqlInstance.executeUpdate(sql) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy index b35f27015..9ac21fba3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -2,28 +2,43 @@ package com.fangjia.script.chongqing import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.DateUtil +import com.fangjia.httpclient.UserAgents import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.paimai.DataUtil import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.google.gson.JsonObject import com.mongodb.BasicDBObject +import com.mongodb.DBObject import groovy.sql.Sql +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import java.util.concurrent.CopyOnWriteArrayList +import java.util.zip.GZIPInputStream class MatchEvaluateRegionTest { + static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/" def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx" + def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx" def total = 1500 static main(args) { @@ -31,14 +46,26 @@ class MatchEvaluateRegionTest { // task.run()//小区匹配 // task.run1()//地图匹配 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 - task.addPrice() +// task.addPrice() // task.addArroundPrice() +// task.addArroundPriceByLat() +// task.addRentRate() // task.addAuthor() +// task.addLevel() +// task.addSorce() +// task.addArroundScore() + + task.pkPrice() + } def run(){ + + def index = 0 ExcelUtils.multiThreadingModify(path, { row -> + index ++ if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,) + println index+" "+row['E'] + def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -47,8 +74,8 @@ class MatchEvaluateRegionTest { row['J'] = jsonObject.get("address") row['K'] = jsonObject.get("propertyType") row['L'] = jsonObject.get("map") -// row['K'] = jsonObject.get("lng") -// row['L'] = jsonObject.get("lat") + row['M'] = jsonObject.get("lng") + row['N'] = jsonObject.get("lat") } } }); @@ -56,8 +83,8 @@ class MatchEvaluateRegionTest { def run1(){ ExcelUtils.multiThreadingModify(path, { row -> - if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null) + if(!row['F'] && row['A']){ + def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString()) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -66,42 +93,81 @@ class MatchEvaluateRegionTest { row['J'] = jsonObject.get("address") row['K'] = jsonObject.get("propertyType") row['L'] = jsonObject.get("map") -// row['K'] = jsonObject.get("lng") -// row['L'] = jsonObject.get("lat") + row['M'] = jsonObject.get("lng") + row['N'] = jsonObject.get("lat") } } }); } + def pkPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['R'] && row['R'].toString().toDouble().toInteger()==1){ + def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd")) + if(!obj){ + row['R'] = '' + row['S'] = "无最新价" + } + } + }); + + } + def addPrice(){ ExcelUtils.multiThreadingModify(path, { row -> if(!row['L']){ def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) if(priceObj){ - row['M'] = priceObj.avg_price - row['N']= (int)(priceObj.avg_price * row['E']) + row['O'] = priceObj.avg_price + // row['N']= (int)(priceObj.avg_price * row['E']) } - def avg_price = findAvgPrice(row['F']) - if(row['M'] && avg_price){ - def weekendPrice = row['M'].toString().toDouble() - def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) - row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I']) + if(sqmetreRentPrice){ + row['P'] = sqmetreRentPrice } + +// def avg_price = findAvgPrice(row['F']) +// if(row['M'] && avg_price){ +// def weekendPrice = row['M'].toString().toDouble() +// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) +// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) +// } } }); } + def addRentRate(){ + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] + def radioMap = [:] + for(def city : citys) { + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) + radioMap.put(city, avg_radio) + } + + ExcelUtils.multiThreadingModify(path, { row -> + def city = row['G'] + if (row['F'] && row['O'] && !row['P']) { + def avg_rent_radio = radioMap.get(city) + if(avg_rent_radio){ + row['P'] = (int)(row['O']/avg_rent_radio) + row['S'] = "租金推算1" + } + } + }) + } + def addArroundPrice(){ ExcelUtils.multiThreadingModify(path, { row -> - if(!row['M']) { - def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D']) + if(!row['O']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E']) if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { def priceObj = arroundObj.result - row['M'] = priceObj.unitPrice - row['N'] = (int) (priceObj.unitPrice * row['E']) - row['Q'] = "周边价格" + row['O'] = priceObj.unitPrice +// row['N'] = (int) (priceObj.unitPrice * row['E']) + row['S'] = "周边价格" } } }); @@ -252,4 +318,295 @@ class MatchEvaluateRegionTest { }); return avg_price } + + def addSorce(){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['F']){ + def score = findDistrictScore(d['F']) + if(score){ + d['R'] = score + } + } + }); + } + + def addArroundScore(){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['R'] && d['F']){ + def dist = null + if(d['F'].toString().size()>16){ + dist = CommonDatautils.findDistrictById(d['F']) + } + + if(!dist){ + dist = new BasicDBObject() + dist.city = d['G'] + dist.lng = d['M'].toString().toDouble()*10000000000L + dist.lat = d['N'].toString().toDouble()*10000000000L + } + if(dist.lat){ + List list = DbUtils.findAroundDistrict(dist) + if (list) { + list = sortByDistance(dist, list); + if(list && list.size()>10){ + list = list[0..9] + } + if(list){ + def scores = [] + for(def s : list){ + def score = findDistrictScore(s._id.toString()) + if(score){ + scores << score + } + } + def arround_score =(int)(scores.sum()/scores.size()) + d['R'] = arround_score + d['T'] = "周边小区评分2" + } + } + } + } + }); + } + + def addArroundPriceByLat(){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){ + def dist = null + if(d['F'].toString().size()>16){ + dist = CommonDatautils.findDistrictById(d['F']) + } + + if(!dist){ + dist = new BasicDBObject() + dist.city = d['G'] + dist.lng = d['M'].toString().toDouble()*10000000000L + dist.lat = d['N'].toString().toDouble()*10000000000L + } + if(dist.lat){ + List list = DbUtils.findAroundDistrict(dist) + if (list) { + list = sortByDistance(dist, list); + if(list && list.size()>10){ + list = list[0..9] + } + if(list){ + def prices = [] + for(def s : list){ + def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + if(prices){ + def arround_price =(int)(prices.sum()/prices.size()) + d['O'] = arround_price + d['S'] = "周边均价3" + } + } + } + } + } + }); + } + + + def sortByDistance(d, list){ + def newList = [] + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + newList << nb + } + //排序 + Collections.sort(newList, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + + return newList; + } + + + + def findDistrictScore(id){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id)) + def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder); + def score = 0 + for(def h : indexObj){ + score = h.total_price_rate + if(score){ + break; + } + } + return score + } + + + def addLevel() { + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + } + + ExcelUtils.multiThreadingModify(path, { row -> + + def address = row['E'] + def city = RegionData.formatCity(row['A']) + + if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } + } + } + } + }) + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + return datas; + } + + def getArroundDistrictPriceByAddress(def city, def address, def levels){ + def json = GaoDeApiUtils.geo(city, address) + def location = null + def detail = new BasicDBObject() + def level = null + def rank = null + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + location = m.location + level = m.level + } + } + if(location){ + def arrs = location.split(",") + detail.lng = arrs[0].toString().toDouble()*10000000000L + detail.lat = arrs[1].toString().toDouble()*10000000000L + } + if(detail.lng){ + detail.city = city + + List list = DbUtils.findAroundDistrictByRadius(detail, 3000) + if(list && list.size()>10){ + list = sortByDistance(detail, list); + try{ + list = list[0..9] + }catch(Exception e){ + println list.size() + } + + } + def arroundAvgPrice = getArroundAvgPrice(list) + if(arroundAvgPrice){ + + def mod = Integer.MAX_VALUE; + for(def d : levels){ + def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice) + if(newMod + def floorsCount = 0; + def buildYearCount = 0 + def householdsCount = 0 + def elevatorNumberCount = 0 + def roomCount = 0 + + def f_floorsCount = 0; + def f_buildYearCount = 0 + def f_householdsCount = 0 + def f_elevatorNumberCount = 0 + def f_roomCount = 0 + + def dist = CommonDatautils.findDistrictById(key) + + if(zRegions.contains(dist.region)){ + z_count++ + }else{ + f_count++ + } + + for(def row : lists){ + + if(zRegions.contains(dist.region)){ + + if(row.floors){ + floorsCount++ + } + if(dist.finish_building_date){ + buildYearCount++ + } + if(row.households){ + householdsCount++ + }else{ + if(row.room_count && row.floors){ + householdsCount++ + } + } + if(row.elevatorNumber && row.elevatorNumber>0){ + elevatorNumberCount++ + } + + if(row.room_count>0){ + roomCount++ + } + + }else{ + if(row.floors){ + f_floorsCount++ + } + if(dist.finish_building_date){ + f_buildYearCount++ + } + if(row.households){ + f_householdsCount++ + }else{ + if(row.room_count && row.floors){ + f_householdsCount++ + } + } + if(row.elevatorNumber && row.elevatorNumber>0){ + f_elevatorNumberCount++ + } + + if(row.room_count>0){ + f_roomCount++ + } + } + + } + + def dist_total_count = lists.size()*0.6 + if(floorsCount >= dist_total_count){ + all_floorsCount ++ + } + + if(buildYearCount >= dist_total_count){ + all_buildYearCount ++ + } + + if(householdsCount >= dist_total_count){ + all_householdsCount ++ + } + + if(elevatorNumberCount >= dist_total_count){ + all_elevatorNumberCount ++ + } + + if(roomCount >= dist_total_count){ + all_roomCount ++ + } + + + if(f_floorsCount >= dist_total_count){ + f_all_floorsCount ++ + } + + if(f_buildYearCount >= dist_total_count){ + f_all_buildYearCount ++ + } + + if(f_householdsCount >= dist_total_count){ + f_all_householdsCount ++ + } + + if(f_elevatorNumberCount >= dist_total_count){ + f_all_elevatorNumberCount ++ + } + if(f_roomCount >= dist_total_count){ + f_all_roomCount ++ + } + } + def floor_prec = DataHandleUtils.getDoubleRound(2, all_floorsCount*100/z_count) + def year_prec = DataHandleUtils.getDoubleRound(2, all_buildYearCount*100/z_count) + def households_prec = DataHandleUtils.getDoubleRound(2, all_householdsCount*100/z_count) + def elevatorNum_prec = DataHandleUtils.getDoubleRound(2, all_elevatorNumberCount*100/z_count) + def roomcount_prec = DataHandleUtils.getDoubleRound(2, all_roomCount*100/z_count) + + + def f_floor_prec = DataHandleUtils.getDoubleRound(2, f_all_floorsCount*100/f_count) + def f_year_prec = DataHandleUtils.getDoubleRound(2, f_all_buildYearCount*100/f_count) + def f_households_prec = DataHandleUtils.getDoubleRound(2, f_all_householdsCount*100/f_count) + def f_elevatorNum_prec = DataHandleUtils.getDoubleRound(2, f_all_elevatorNumberCount*100/f_count) + def f_roomcount_prec = DataHandleUtils.getDoubleRound(2, f_all_roomCount*100/f_count) + + println "floor "+ floor_prec + println "year "+ year_prec + println "households "+ households_prec + println "elevator "+ elevatorNum_prec + + def rets = [] + rets << ["城区","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观"] + rets << ["主城区",100,100,100,0,floor_prec,0,year_prec,elevatorNum_prec,roomcount_prec,households_prec,0] + rets << ["非主城区",100,100,100,0,f_floor_prec,0,f_year_prec,f_elevatorNum_prec,f_roomcount_prec,f_households_prec,0] + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "楼栋") + } + + + def houseLoudongStat2(){ + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name"]) + def z_count = 0 + def f_count = 0 + + def z_directionCount = 0 + def f_directionCount = 0 + + def f_sightCount = 0 + def z_sightCount = 0 + def index = 0 + for(def dist : dists){ + index ++ + def rets = getAllHouseLoudong2(dist._id.toString()) + println index +" "+rets.size() + if(rets){ + def directionCount = 0 + def sightCount = 0 + for(def d : rets){ + if(d.direction){ + directionCount ++ + } + if(d.sight){ + sightCount ++ + } + } + + def total = rets.size()*0.6 + if(zRegions.contains(dist.region)){ + z_count++ + + if(directionCount>=total){ + z_directionCount++ + } + if(sightCount >=total){ + z_sightCount++ + } + }else{ + f_count++ + + if(directionCount>=total){ + f_directionCount++ + } + if(sightCount >=total){ + f_sightCount++ + } + } + } + } + + def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count) + def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count) + + def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count) + def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count) + def rets = [] + rets << ["城区","户号","户ID","主朝向","景观","户型结构"] + rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100] + rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100] + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户1") + } + + def houseLoudongStat(){ + def map = getAllHouseLoudong() + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + + def z_count = 0 + def f_count = 0 + + def z_directionCount = 0 + def f_directionCount = 0 + + def f_sightCount = 0 + def z_sightCount = 0 + map.each{key,list-> + def dist = CommonDatautils.findDistrictById(key) + def directionCount = 0 + def sightCount = 0 + for(def d : list){ + if(d.direction){ + directionCount ++ + } + if(d.sight){ + sightCount ++ + } + } + + def total = list.size()*0.6 + if(zRegions.contains(dist.region)){ + z_count++ + + if(directionCount>=total){ + z_directionCount++ + } + if(sightCount >=total){ + z_sightCount++ + } + }else{ + f_count++ + + if(directionCount>=total){ + f_directionCount++ + } + if(sightCount >=total){ + f_sightCount++ + } + } + } + + def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count) + def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count) + + def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count) + def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count) + def rets = [] + rets << ["城区","户号","户ID","主朝向","景观","户型结构"] + rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100] + rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100] + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户") + } + + def getAllHouseLoudong(){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("district_id", new BasicDBObject(QueryOperators.EXISTS, true)) + + def keys = new BasicDBObject() + keys.put("district_id",1) + keys.put("direction",1) + keys.put("sight",1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def retMap = [:] + for(def d : dbcursor){ + def districtId = d.district_id + if(districtId){ + def lists = retMap.get(districtId) + if(!lists){ + lists = [] + retMap.put(districtId, lists) + } + lists << d + } + } + return retMap; + } + + + def getAllHouseLoudong2(districtId){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("district_id", districtId) + + def keys = new BasicDBObject() + keys.put("district_id",1) + keys.put("direction",1) + keys.put("sight",1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def lists = [] + for(def d : dbcursor){ + lists << d + } + return lists; + } + + def getAllLouDong(){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = LOUDONG_LINE_COLL.find(q).batchSize(5000) + def retMap = [:] + for(def d : dbcursor){ + def districtId = d.id_in_seaweed + if(districtId){ + def lists = retMap.get(districtId) + if(!lists){ + lists = [] + retMap.put(districtId, lists) + } + lists << d + } + } + return retMap; + } + + def updteHoledCount(){ + def datas = findAllLouDong() + def totalCount = new AtomicLong() + println datas.size() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i2){ + q.put("unit_num", arrs[2]) + } + + def udata = new BasicDBObject() + udata.put("room_count", max_room) + LOUDONG_LINE_COLL.update(q, new BasicDBObject('$set', udata), false, true); + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + } + + def findAllLouDong(){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = LOUDONG_LINE_COLL.find(q) + def keys = new HashSet() + for(def d : dbcursor){ + def districtId = d.id_in_seaweed + def key = districtId +"##"+d.house_num + if(d.unit_num && d.unit_num!="0"){ + key = key + "##"+d.unit_num + } + if(!keys.contains(key)){ + keys << key + } + } + return keys + } + + def findHouseLouDongRoom(def arrs){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("district_id", arrs[0]) + q.put("house_num", arrs[1]) + if(arrs.size()>2){ + q.put("unit_num", arrs[2]) + } + def dbcursor = HOUSE_LOUDONG_COLL.find(q) + def max_room = 0 + for(def d : dbcursor){ + def room = null + if(d.room_num && d.room_num.contains("-")){ + room = d.room_num.substring(d.room_num.indexOf("-")+1, d.room_num.length()) + }else if(d.room_num && d.room_num.contains("0")){ + room = d.room_num.substring(d.room_num.toString().lastIndexOf("0")+1, d.room_num.length()) + } + if(room && room.isNumber()){ + if(room.toInteger()>max_room){ + max_room = room.toInteger() + } + } + } + return max_room; + } + def removeDelete(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name", "name_cat"]) + for(def obj : dists){ + def name_cat = obj.name_cat; + if(name_cat==12){ + def count = CommonDatautils.findDistrictCount(obj.city, obj.region, obj.name, "district") + if(count>1){ + println obj.name +" "+count + DbUtils.seaweed.remove(new BasicDBObject("_id", obj._id)) + } + } + } } def regionDistStat(){ @@ -36,25 +494,25 @@ class PocStat { for(def region : set){ def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district") - def regionCount = 0 - def regionHouseLDCount = 0 - for(def d : dists){ - if(d.region == region){ - def loudongCount = CommonDatautils.findLoudong(d._id.toString()) - if(loudongCount && loudongCount>0){ - regionCount = regionCount+ loudongCount - } - - def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString()) - if(houseLouDongCount && houseLouDongCount>0){ - regionHouseLDCount = regionHouseLDCount + houseLouDongCount - } - } - } - rows << [region, count, regionCount, regionHouseLDCount] +// def regionCount = 0 +// def regionHouseLDCount = 0 +// for(def d : dists){ +// if(d.region == region){ +// def loudongCount = CommonDatautils.findLoudong(d._id.toString()) +// if(loudongCount && loudongCount>0){ +// regionCount = regionCount+ loudongCount +// } +// +// def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString()) +// if(houseLouDongCount && houseLouDongCount>0){ +// regionHouseLDCount = regionHouseLDCount + houseLouDongCount +// } +// } +// } + rows << [region, count] println region +" 完成" } - ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计") + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计1") } def sampleStat(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy index 7cb4b8ae2..29a2403b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy @@ -6,12 +6,17 @@ import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql import org.bson.types.ObjectId import java.util.concurrent.CopyOnWriteArrayList class PocTest { + def total = 3259 + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { def task = new PocTest(); task.run(); @@ -27,17 +32,21 @@ class PocTest { * @return */ def run(){ - def inputfile = "D:\\数据\\重庆农商行\\poc\\重庆农商行.xlsx" - match(inputfile) - addPrice(inputfile) - addArroundPrice(inputfile) + def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls" +// match(inputfile) +// addPrice(inputfile) +// addArroundPrice(inputfile) +// standard(inputfile) +// addLoudongNum(inputfile) + + +// addAuthor(inputfile) + +// pkPrice(inputfile) + +// addPpeValue(inputfile) + // matchMap(inputfile) - - standard(inputfile) - addLoudongNum(inputfile) - - - // addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx") // // createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx") @@ -47,6 +56,36 @@ class PocTest { // createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx") } + def addPpeValue(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['F'] && row['R']){ + def avg_price = findAvgPrice(row['R']) + if(avg_price){ + def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price) + row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + } + + def pkPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['R'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString()) + if(jsonObject != null && jsonObject.code==200) { + def result = jsonObject.result + if(result){ + row['Y'] = result.get("unitPrice") + } + } + } + } + }) + } + def match(path){ ExcelUtils.multiThreadingModify(path, { row -> @@ -54,7 +93,7 @@ class PocTest { def city = row['B'] def address = row['D'] def region = row['C'] - println "city:"+city+" region:"+region+" address:"+address + println "[match] city:"+city+" region:"+region+" address:"+address def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null) if(jsonObject != null && jsonObject.precise!=-1) { row['Q'] = jsonObject.get("name") @@ -95,8 +134,9 @@ class PocTest { ExcelUtils.multiThreadingModify(path, { row -> def commid = row['R'] if(commid){ - def dist = CommonDatautils.findDistrictById(commid) + def dist = CommonDatautils.findDistrictById(commid.trim()) if(dist){ + row['Q'] = dist.name def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) if(priceObj){ row['F'] = priceObj.avg_price @@ -106,6 +146,8 @@ class PocTest { }) } + + def addArroundPrice(path){ ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']) { @@ -156,8 +198,8 @@ class PocTest { if (jsonObject && jsonObject.result) { if(jsonObject.result.size()>=1){ def obj = jsonObject.result[0] - row['T'] = obj.id - row['S'] = obj.houseNumber + row['T'] = obj.id + row['S'] = obj.houseNumber } } @@ -172,6 +214,7 @@ class PocTest { } } } + }) } @@ -590,4 +633,34 @@ class PocTest { } ExcelUtils.write(output, rows, "成交案例") } + + def addAuthor(def path){ + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['B']){ + if(index < avg){ + row['X'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index af552d3d4..b8e3cd5a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3079,18 +3079,26 @@ class CommonDatautils { queryCondition.put("region", region); queryCondition.put("district_name", name); queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate)); - DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1) + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) DBObject dbo; + def prices = [] + def index = 0 while(cur.hasNext()) { - dbo = cur.next() - def price = dbo.get("avg_price"); - if(price){ - break; + def obj = cur.next() + if(index == 0){ + dbo = obj } + prices << obj.get("avg_price"); + index++ } - return dbo; + if(prices){ + def avg_price = (int)(prices.sum()/prices.size()) + dbo.avg_price = avg_price + return dbo; + } + return null; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index e8cd3476a..bb7b11f46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -6,10 +6,16 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jusfoun.JusfounCommon +import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.io.FileUtils +import org.bson.types.ObjectId +import java.text.DecimalFormat import java.util.concurrent.TimeUnit /** @@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit class DistrictPriceExportBatch extends BaseBatch{ def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def charset = "GBK" + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { runBatch(new DistrictPriceExportBatch(), args) } @@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{ def ret = createDatas("福州") + saveToDB(ret) + + def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" if(CommonDatautils.isWindow()){ @@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{ def createDatas(def city){ def districts = CommonDatautils.findAllDistricts(city, "district", null); - - def lastDate = new Date() - def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) - + def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" + def dataStr = DateUtils.add(endDateStr, -30) + def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr) + def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd") // List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) // def map2 = [:] // for(def s : list2) { @@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{ // map2.put(key, row[5]) // } + def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate) def result = [] for(def rMap : districts){ + def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) if(dbo){ + def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString()) + def curPrice = dbo.get("avg_price") + if(lastMonthPrice && curPrice){ + def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice + if(Math.abs(markup)>3){ + def randMarkup = nextDouble(0, 3) + if(markup < 0){ + markup = randMarkup*(-1) + }else{ + markup = randMarkup + } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) + } + } + def m = [] m << dbo.get("_id").toString()//均价ID m << rMap.get("_id").toString()//楼盘ID m << rMap.city m << rMap.region m << rMap.name - m << dbo.get("avg_price") + m << curPrice // def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name) // if(lastMonthPrice && dbo.get("avg_price")){ // lastMonthPrice = lastMonthPrice.toInteger() @@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{ return result } + + def findFzDistrictStat(city,lastMonthDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("month", lastMonthDate) + def dbcursor = FZ_DISTRICT_STAT_COLL.find(q) + def priceMap = [:] + for(def d : dbcursor){ + priceMap.put(d.comm_id, d.avg_price) + } + return priceMap + } + def export(def fileName, def datas){ def headers = ["id","comm_id","city","district","name","avg_price","year_month"] writeCSV(headers, fileName, datas); @@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{ file.append(System.getProperty("line.separator")); } } + + //随机浮点数 + public static double nextDouble(final double min, final double max) throws Exception { + + DecimalFormat df = new DecimalFormat("#.00"); + if (max < min) { + throw new Exception("min < max"); + } + if (min == max) { + return min; + } + return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble()))); + } + + def saveToDB(rets){ + for(def r : rets){ + def q = new BasicDBObject() + q.put("month", curDate.toInteger()) + q.put("comm_id", r[1]) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if(obj){ + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("avg_price", r[5]) + FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + def o = new BasicDBObject() + o.put("_id", new ObjectId(r[0])) + o.put("comm_id", r[1]) + o.put("city", r[2]) + o.put("region", r[3]) + o.put("name", r[4]) + o.put("avg_price", r[5].toInteger()) + o.put("month", r[6].toInteger()) + FZ_DISTRICT_STAT_COLL.save(o) + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index ba848d886..81c5624d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -8,7 +8,9 @@ import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBCursor import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import org.apache.commons.io.FileUtils +import org.bson.types.ObjectId /** * 福州数据结果统计 @@ -17,16 +19,42 @@ class FzDataResultStat { public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); task.regionPriceTrendRun() +//// task.margeExcel() +// task.saveCsvData() + } + + def margeExcel(){ + + def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1) + + def map2 = [:] + for(def s : list2){ + def key = s["A"]+"##"+s["B"]+"##"+s["C"] + map2.put(key, s["D"]) + } + + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row-> + if(row["A"]){ + def key = row["A"]+"##"+row["B"]+"##"+row["C"] + def price = map2.get(key) + if(price){ + row['L'] = price + } + } + }) } def priceMarkupRun(){ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def map1 = [:] for(def s : list1){ def row = s.split("@\\|@") @@ -137,16 +165,13 @@ class FzDataResultStat { queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate)); DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) - def price = 0; + def prices = []; while(cur.hasNext()) { DBObject dbo = cur.next() - price = dbo.get("avg_price"); - if(price){ - break; - } + prices << dbo.get("avg_price"); } - - return price; + def avg_price = (int)(prices.sum()/prices.size()) + return avg_price; } @@ -165,4 +190,45 @@ class FzDataResultStat { weekends = weekends.reverse(); return weekends; } + + + def saveCsvData(){ + def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)") + if(files){ + for(def filepath : files){ + List list1= FileUtils.readLines(filepath) + println filepath + for(def s : list1){ + def row = s.split("@\\|@") + def o = new BasicDBObject() + o.put("_id", new ObjectId(row[0])) + o.put("comm_id", row[1]) + o.put("city", row[2]) + o.put("region", row[3]) + o.put("name", row[4]) + o.put("avg_price", row[5].toInteger()) + o.put("month", row[6].toInteger()) + FZ_DISTRICT_STAT_COLL.save(o) + + } + } + } + } + + def static getAllFiles(def path) { + + File file = new File(path); + File[] tempList = file.listFiles(); + + def files = []; + for (int i = 0; i < tempList.length; i++) { + if (tempList[i].isFile()) { + files << tempList[i]; + } + if (tempList[i].isDirectory()) { + System.out.println("文件夹:" + tempList[i]); + } + } + return files; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 5a6b21aac..d554c0376 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -137,7 +137,7 @@ ExcelUtils.write("d:\\ def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -DingDingMessageUtil.sendTextMessage(access_token, s) +//DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy index 244497f17..12f241dd6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy @@ -112,22 +112,27 @@ class TradeExportBatch extends BaseBatch{ // q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate)) def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000); def rets = [] + def urls = [] if (cursor != null) { while (cursor.hasNext()) { DBObject dbo = cursor.next(); def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name); if(district){ - def ret = []; - ret << dbo._id.toString() - ret << district._id.toString() - ret << dbo.city - ret << dbo.region - ret << dbo.district_name - ret << dbo.avg_price - ret << dbo.area - ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") - ret << dbo.total_price - rets << ret; + if(!urls.contains(dbo.url)){ + urls << dbo.url + + def ret = []; + ret << dbo._id.toString() + ret << district._id.toString() + ret << dbo.city + ret << dbo.region + ret << dbo.district_name + ret << dbo.avg_price + ret << dbo.area + ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") + ret << dbo.total_price + rets << ret; + } } } println "成交总计:"+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index 4601c3c60..d55350598 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -1,10 +1,13 @@ package com.fangjia.script.gj.crawl import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import net.sf.json.JSONObject import org.jsoup.Jsoup import org.openqa.selenium.WebDriver @@ -12,16 +15,21 @@ import org.openqa.selenium.chrome.ChromeDriver class AnjukeAppCrawl { + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + public static def SOURCE = "安居客"; DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") static main(args) { def task = new AnjukeAppCrawl(); - task.run(); +// task.run(); // task.run2() + + task.addThirdPrice() } def run2(){ - def url = "https://m.anjuke.com/fz/community/" + def url = "https://m.anjuke.com/sh/community/" System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() @@ -95,17 +103,28 @@ class AnjukeAppCrawl { def run(){ + //上海:11,北京:14 //"仓山":4071, - def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384] - map.each{region, area_id-> + // def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384] + // def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20] + def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678] + def cityMap = ["北京":regionMap] + def cityId = 14 + regionMap.each{region, area_id-> for(def i=1;i<40;i++){ def stop = false; def all_stop = false + def error_count=0 while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380 //城口 - def flagMap = getInfo(stop, i, area_id) + def flagMap = getInfo(stop, i, area_id, cityId, "上海") stop = flagMap.stop all_stop = flagMap.all_stop + error_count++ + if(error_count>=20){ + stop = true + all_stop = true + } } if(all_stop){ break; @@ -114,17 +133,17 @@ class AnjukeAppCrawl { } } - def getInfo(def stop, def i, def area_id){ + def getInfo(def stop, def i, def area_id, def cityId, def city){ def map = [:] - def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1" + def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1" - def header = [:] + def header = [:] header.put("accept","application/json, text/plain, */*") header.put("accept-encoding","gzip, deflate, br") header.put("accept-language","zh-CN,zh;q=0.9") header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") - header.put("referer","https://m.anjuke.com/fz/community/") + header.put("referer","https://m.anjuke.com/sh/community/") header.put("sec-fetch-dest","empty") header.put("sec-fetch-mode","cors") header.put("sec-fetch-site","same-origin") @@ -134,6 +153,7 @@ class AnjukeAppCrawl { def ret = ABuYunUtil.doGet(url, null, null, "utf-8") def all_stop = false; + def error_count = 0 if (ret) { JSONObject jsonObj = JSONObject.fromObject(ret); if(jsonObj && jsonObj.status=="ok" ){ @@ -143,7 +163,7 @@ class AnjukeAppCrawl { println "【${i}】 "+communities.size() +" "+url if(communities.size()>0){ - save("福州", communities) + save(city, communities) Thread.sleep(10000) stop = true; }else{ @@ -154,6 +174,7 @@ class AnjukeAppCrawl { }else{ stop = false; + error_count ++ println url +" 无数据" } @@ -195,4 +216,42 @@ class AnjukeAppCrawl { q.put("jumpAction",url) return detail.findOne(q); } + + def addThirdPrice(){ + def q = new BasicDBObject() + def dbcursor = detail.find(q) + def index = 0 + for(def d : dbcursor){ + index++ + def city = d.base.cityName + def region = d.base.areaName + def name = d.base.name + def obj = findByParams(city, region, name); + if(!obj){ + def data = new BasicDBObject() + data.put("district_id",d.base.id) + data.put("city", city) + data.put("region", region ) + data.put("name", name) + data.put("address",d.base.address) + data.put("avg_price", d.priceInfo.price) + data.put("source", SOURCE) + data.put("s_date", SHORT_DATE) + data.put("c_date", d.c_date) + thirdPartyColl.save(data) + println index +" "+data + } + } + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", SOURCE) + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy index 1954d135c..85060a709 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy @@ -22,9 +22,9 @@ class AnjukeToDB { static main(args) { def task = new AnjukeToDB(); -// task.run(); + task.run(); // task.addNoMatch() - task.updateEstateType() +// task.updateEstateType() // task.gaodeRangeData() } @@ -315,16 +315,16 @@ class AnjukeToDB { } } def run(){ - def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1) for(def data : datas){ - if(data['K']=="芝芝"){ +// if(data['K']=="镕杰"){ def flag = data['M'] if(flag && flag=="新增"){ - add(data) +// add(data) }else{ update(data) } - } +// } } } @@ -445,8 +445,6 @@ class AnjukeToDB { } if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){ addr_alias << obj.base.address - } - if(addr_alias!=dist.addr_alias && addr_alias.size()>0){ udata.put("addr_alias", addr_alias) } } @@ -555,8 +553,13 @@ class AnjukeToDB { def findAnjukeDistrictDetail(anjukeId){ def q = new BasicDBObject() - q.put("_id",new ObjectId(anjukeId)) - return detail.findOne(q) + try{ + q.put("_id",new ObjectId(anjukeId)) + return detail.findOne(q) + }catch(Exception e){ + print "[查询出错]anjukeId:"+anjukeId + } + return null } def nameFileter(def region, def name){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 71657eea3..cbee24228 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -172,7 +172,8 @@ class NameProcess { if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){ return null } - if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){ + //|| name.endsWith("组") + if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){ return null } if(name.contains("、")||name.contains("|")||name.contains(".")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index bd387ed77..3a572c7ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -71,7 +71,8 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def citys = CommonDatautils.findAllCity(); + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] + //def citys = CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def code = cityCodeDatas.get(city) @@ -86,7 +87,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - // calcute(city, dists, code, city_avg_total_price) + calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy index 6b4872d19..f42431df6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy @@ -21,10 +21,58 @@ class MatchTest { def task = new MatchTest() // task.run() // task.addNum() - task.dongguang() +// task.dongguang() // task.meituanDel() + + task.addLevel() } + + + def addLevel() { + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + } + + + def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + + def name = row['D'] + def address = row['E'] + def city = RegionData.formatCity(row['A']) + + if(name && !row['O']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } +// if(rMap.level){ +// row['Q'] = rMap.level +// } + } + } + } + + }) + } + + def meituanDel(){ def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index a97cb7669..60ae682b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -193,7 +193,7 @@ public class BeiKeCrawlByChrome { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() } } @@ -215,7 +215,8 @@ public class BeiKeCrawlByChrome { detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") } detail.source = "贝壳找房" - def dbo = findTrade(detail) + def detailUrl = e.select(".title a").attr("href") + def dbo = findTrade(detailUrl) if (!dbo) { def houseInfo = e.select(".houseInfo").text() def houseInfos = houseInfo.split("\\|") @@ -228,7 +229,7 @@ public class BeiKeCrawlByChrome { detail.height = floorStr.toInteger() } } - detail.url = e.select(".title a").attr("href") + detail.url = detailUrl detail.create_date = new Date() detail.m_date = new Date() println detail @@ -313,8 +314,8 @@ public class BeiKeCrawlByChrome { def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") if (roomStr) { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger() } detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") @@ -336,13 +337,14 @@ public class BeiKeCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def detailUrl){ def q = new BasicDBObject() q.put("city", detail.city) - q.put("region", detail.region) - q.put("district_name", detail.district_name) - q.put("area", detail.area) - q.put("total_price", detail.total_price) +// q.put("region", detail.region) +// q.put("district_name", detail.district_name) +// q.put("area", detail.area) +// q.put("total_price", detail.total_price) + q.put("url", detailUrl) q.put("trade_date", detail.trade_date) q.put("source", detail.source) def dbo = DEAL_PRICE_COLL.findOne(q) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index cae68c08d..63ecf3c2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,11 +34,11 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() -// task.addDealRegion(); -// task.standardRegion() - task.updadteDealPriceField() + task.addDealRegion(); + task.standardRegion() +// task.updadteDealPriceField() } def updadteDealPriceField(){ @@ -278,7 +278,7 @@ public class LianjiaCrawlByChrome { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() } } @@ -398,8 +398,8 @@ public class LianjiaCrawlByChrome { def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") if (roomStr) { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger() } detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 8d971ee8a..95bfde522 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -16,9 +16,11 @@ import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.bson.types.ObjectId import java.text.DecimalFormat +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern @@ -34,6 +36,9 @@ class CZDataBatch extends BaseBatch{ DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com") //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); //final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls"); + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + static main(args) { runBatch(new CZDataBatch(), args); } @@ -113,25 +118,44 @@ class CZDataBatch extends BaseBatch{ } def districtRun(def districts){ - def total = districts.size(); - def index = 0; - ExecutorService executorService = Executors.newFixedThreadPool(20) - + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(total) for(def i=0;i +// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州", + def citys = ["唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","北京","衡阳","南京","滨州","张家口","商丘","赣州","湖州","安庆","聊城","枣庄","阜阳","西安","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳"] + for(def city : citys){ + def str = cityList.get(city) for(def i=0;i<1;i++){ - if(citys.contains(city)){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); } } } +// cityList.each{city, str-> +// for(def i=0;i<1;i++){ +// if(citys.contains(city)){ +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// } +// } +// } +// } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 1a45be219..f0186187b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -266,7 +266,7 @@ public class OpenApiClient { params.put("unitNumber", unitNum); params.put("roomNumber", roomNum); params.put("token", TOKEN); - String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") @@ -288,6 +288,38 @@ public class OpenApiClient { return null } + + public static JSONObject webankEvaluate(String city, String region, String address, String area){ + try{ + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("address", address); + params.put("area", area.toString()); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${city} ${address} 程序调用异常") + } + } + + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standard2Action(String city, String address) { try { Map params = new HashMap(); @@ -706,7 +738,7 @@ public class OpenApiClient { params.put("startDate", startDate) params.put("limit", "100") try { - String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8") JSONObject result = JSONObject.fromObject(content); return result } catch (Exception e) { From eea00b9b233c6144380fb2f95ee136fd79e3d791 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 27 May 2022 15:49:53 +0800 Subject: [PATCH 75/83] update --- .../com/fangjia/script/fuzhou/TradeExportBatch.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy index 12f241dd6..cd6252fb6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy @@ -112,15 +112,16 @@ class TradeExportBatch extends BaseBatch{ // q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate)) def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000); def rets = [] - def urls = [] + def keys = [] if (cursor != null) { while (cursor.hasNext()) { DBObject dbo = cursor.next(); def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name); if(district){ - if(!urls.contains(dbo.url)){ - urls << dbo.url - + def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") + def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr + if(!keys.contains(key)){ + keys << key def ret = []; ret << dbo._id.toString() ret << district._id.toString() @@ -129,7 +130,7 @@ class TradeExportBatch extends BaseBatch{ ret << dbo.district_name ret << dbo.avg_price ret << dbo.area - ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") + ret << tradeDateStr ret << dbo.total_price rets << ret; } From b07c07487c92ec452128438153f06656a80ce38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Mon, 30 May 2022 10:41:02 +0800 Subject: [PATCH 76/83] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/address/processor/AddressProcessors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java index 9097a3a63..d1a15d156 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java @@ -28,8 +28,8 @@ public class AddressProcessors{ addressProcessors.add(new ProvinceProcessor()); addressProcessors.add(new CityProcessor()); addressProcessors.add(new DistrictProcessor()); - addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new TownshipProcessor()); + addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new StageAndPartitionProcessor()); addressProcessors.add(new DirectionProcessor()); addressProcessors.add(new OffsetProcessor()); From 95cd48bc06c71c334b287a065665e8dda9ce6aed Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 30 May 2022 20:59:51 +0800 Subject: [PATCH 77/83] =?UTF-8?q?=E6=8A=93=E5=8F=96=E5=AE=A1=E6=A0=B8?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/GetPoiFromExcel.groovy | 19 ++++++++++++------- .../temp/shenjian/util/GaoDeUtils.groovy | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy index bd49875b0..f9b073743 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy @@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils @@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong class GetPoiFromExcel { - static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi" static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; static void main(String[] args) { AtomicLong atomicLong = new AtomicLong(0) //List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); - List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false); + List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false); for (File file : fileList) { List> seaweeds = ExcelUtils.read(file.getPath()) for (List infoTemp : seaweeds) { @@ -28,12 +32,13 @@ class GetPoiFromExcel { executorService.submit(new Runnable() { @Override void run() { - String city = info.get(0) - String region = info.get(1) - String seaweedName = info.get(2) - List pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302"); + String city = info.get(1) + String region = info.get(2) + String seaweedName = info.get(3) + List pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) HttpUtils.postForJson(URL, jsonStr) atomicLong.incrementAndGet() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index ea456aa50..6b880c5ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger class GaoDeUtils { // 新获取key - def static GAO_DE_AK = ["4ccfe3c898a9489884bd2bcb1ef6dcde", "4e9592208336433ee1b7aa4d1d434418"] + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"] def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ From e97f28e23965fd13bf8942046682b083eab006e4 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 1 Jun 2022 10:24:53 +0800 Subject: [PATCH 78/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/didata/MatchFromExcel.groovy | 26 +++++++++++++++++++ .../temp/shenjian/poi/GetPoiFromExcel.groovy | 6 ++--- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy new file mode 100644 index 000000000..20de59c5f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy @@ -0,0 +1,26 @@ +package com.fangjia.script.temp.shenjian.didata + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +/** + * 从excel中匹配出 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row -> + def city = row['A'] + def region = row['B'] + def district_name = row['C'] + + def gaodeRegion = row['H'] + def gaodeName = row['I'] + + def jsonObject = OpenApiClient.addressMatch(city, null, "district", false, region, district_name, "strict") + if (jsonObject != null && jsonObject.precise != -1) { + row['D'] = jsonObject.get("name") + } + + def jsonObject2 = OpenApiClient.addressMatch(city, null, "district", false, gaodeRegion, gaodeName, "strict") + if (jsonObject != null && jsonObject.precise != -1) { + row['J'] = jsonObject.get("name") + } +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy index f9b073743..235d779e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy @@ -32,9 +32,9 @@ class GetPoiFromExcel { executorService.submit(new Runnable() { @Override void run() { - String city = info.get(1) - String region = info.get(2) - String seaweedName = info.get(3) + String city = info.get(0) + String region = info.get(1) + String seaweedName = info.get(2) List pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); if (CollectionUtils.isNotEmpty(pois)) { List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); From 620bf0f8d8d71619310b0dd2193d4543eccdbd14 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 1 Jun 2022 10:28:19 +0800 Subject: [PATCH 79/83] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/didata/MatchFromExcel.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy index 20de59c5f..6cf7730ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy @@ -14,13 +14,15 @@ ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识 def gaodeRegion = row['H'] def gaodeName = row['I'] - def jsonObject = OpenApiClient.addressMatch(city, null, "district", false, region, district_name, "strict") + def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict") if (jsonObject != null && jsonObject.precise != -1) { row['D'] = jsonObject.get("name") } - def jsonObject2 = OpenApiClient.addressMatch(city, null, "district", false, gaodeRegion, gaodeName, "strict") - if (jsonObject != null && jsonObject.precise != -1) { - row['J'] = jsonObject.get("name") + if (gaodeRegion && gaodeName) { + def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict") + if (jsonObject2 != null && jsonObject2.precise != -1) { + row['J'] = jsonObject2.get("name") + } } }); From cae5b302d3dcf9f5ee6139a30e6bb317b6072394 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 1 Jun 2022 13:46:10 +0800 Subject: [PATCH 80/83] update --- .../BigdataCommunityBatch.groovy | 21 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../com/fangjia/script/gj/OpenApiUtils.groovy | 1 + .../script/gj/ccb/CcbCityPriceStat.groovy | 70 ++-- .../script/gj/crawl/DealPriceUtils.groovy | 347 ++++++++++++++++++ .../script/gj/crawl/ZhugeFangDistCrawl.groovy | 167 +++++++++ .../gj/location/DistrictMapMatch.groovy | 123 +++++++ .../script/jiangnanblank/CZDataBatch.groovy | 28 +- .../thirdPriceCrawl/DataStandardBatch.groovy | 3 +- .../fangjia/script/util/CommonUtils.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 28 ++ 11 files changed, 744 insertions(+), 48 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2f8fe431..6cd17549e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{ }else{ def jsonObj = null if(dist.address){ - jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) +// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) } - if(jsonObj && jsonObj.searchAddress){ - ret << jsonObj.searchAddress.street - ret << jsonObj.searchAddress.streetNumber + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street + }else{ + ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ + ret << "" + } + }else{ ret << "" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index d554c0376..7e5074c7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -137,7 +137,7 @@ ExcelUtils.write("d:\\ def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -//DingDingMessageUtil.sendTextMessage(access_token, s) +DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 77239e3ea..e4b621c99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -164,6 +164,7 @@ class OpenApiUtils { if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ return null; } + address = address.trim().replace(" ","") String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" List poiList = null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy index c60a811b5..19709dd22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -18,44 +18,62 @@ class CcbCityPriceStat { task.run() } def run(){ - def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] def rets = [] + def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd") + for(def city : citys){ - def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) - def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) - def cityRets = [] - for(def dist : districts){ - def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") - def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) - if(tradeAvgPrice && weekendPrice){ - def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] - cityRets << row - println row.toString() + def matchTotalCount = findMatchTradeCount(city, startDate, endDate) + println city +" "+matchTotalCount + if(matchTotalCount){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } } + print city +" "+ cityRets.size() + rets.addAll(cityRets) } - print city +" "+ cityRets.size() - rets.addAll(cityRets) + } println "总量:"+rets.size() - ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有") + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count } def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() - q.put("city", dist.city) - q.put("region", dist.region) - - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - q.put("district_name", new BasicDBObject('$in', dist.alias)) - }else{ - q.put("district_name", dist.name) - } - +// q.put("city", dist.city) +// q.put("region", dist.region) +// +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// q.put("district_name", new BasicDBObject('$in', dist.alias)) +// }else{ +// q.put("district_name", dist.name) +// } + q.put("match_id", dist._id.toString()) q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) def dbcorsor = DEAL_PRICE_COLL.find(q) def avg_price = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy new file mode 100644 index 000000000..02fc9458d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy @@ -0,0 +1,347 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * 鎴愪氦鏁版嵁瑙f瀽宸ュ叿绫� + */ +class DealPriceUtils { + + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["涓滃寳", "瑗垮寳", "涓滃崡", "瑗垮崡", "鍗楀寳", "涓滆タ", "涓�", "鍗�", "瑗�", "鍖�"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓(瀹鎴�)"), { String str -> + str = str.replaceAll("瀹鎴�", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍘�"), { String str -> + str = str.replace("鍘�", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍗�"), { String str -> + str = str.replace("鍗�", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?涓�"), { String str -> + str = str.replace("涓�", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/銕�"), { String str -> + str = str.replace("鍏�/銕�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/m虏"), { String str -> + str = str.replace("鍏�/m虏", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/骞�"), { String str -> + str = str.replace("鍏�/骞�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�"), { String str -> + str = str.replace("鍏�", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞崇背"), { String str -> + str = str.replace("骞崇背", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞虫柟绫�"), { String str -> + str = str.replace("骞虫柟绫�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞�"), { String str -> + str = str.replace("骞�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?銕�"), { String str -> + str = str.replace("銕�", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m虏"), { String str -> + str = str.replace("m虏", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("鍏盶\d+灞�"), { String str -> + str = str.replace("鍏�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("鎬籠\d+灞�"), { String str -> + str = str.replace("鎬�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("绗琝\d+(灞倈妤�)"), { String str -> + str = str.replace("绗�", "").replaceAll("灞倈妤�", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("姣涘澂", ["姣涘澂", "娓呮按", "鏃犺淇�", "鐙爧姣涘澂", "姣涜儦", "鐧藉澂", "鏃ц", "鍧�", "鍘熸埧", "姣涘潖"]) + FITMENTS_TYPES_MAPS.put("绠�瑁�", ["绠�瑁�", "鏅�氳淇�", "鏅��", "绠�鍗曡淇�", "绠�鏄撹淇�", "鏅", "涓�鑸淇�", "涓�鑸�", "绠�", "绮楄淇�"]) + FITMENTS_TYPES_MAPS.put("绮捐", ["绮捐", "鍏ㄨ淇�", "鍏ㄨ", "涓瓑瑁呬慨", "涓", "涓。瑁呬慨", "绮捐嚧瑁呬慨", "涓。", "涓瓑", "绮�", "涓慨", "浼樿川瑁呬慨", "鏃跺皻瑁呬慨", "鑹", "涓珮鎸�"]) + FITMENTS_TYPES_MAPS.put("璞", ["璞崕瑁呬慨", "楂樻。", "璞崕", "璞", "楂樼骇瑁呬慨", "楂樿", "濂緢"]) + + } + /** + * 瑙f瀽鎴愪氦鏃ユ湡 + * @param content + * @return + */ + static Date parseDealDate(String content) { + return parse(content, DEAL_DATE_RESOLVERS) + } + + /** + * 瑙f瀽瀹ゆ暟 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + /** + * 瑙f瀽鎬讳环 + * @param content + * @return + */ + static Integer parseTotalPrice(String content) { + return parse(content, TOTAL_PRICE_RESOLVERS) + } + /** + * 瑙f瀽鍗曚环 + * @param content + * @return + */ + static Integer parseUnitPrice(String content) { + return parse(content, UNIT_PRICE_RESOLVERS) + } + /** + * 瑙f瀽闈㈢Н + * @param content + * @return + */ + static Double parseArea(String content) { + return parse(content, AREA_RESOLVERS) + } + /** + * 瑙f瀽妤煎眰 + * @param content + * @return + */ + static Integer parseFloor(String content) { + return parse(content, FLOOR_RESOLVERS) + } + /** + * 瑙f瀽妤奸珮 + * @param content + * @return + */ + static Integer parseHeight(String content) { + return parse(content, HEIGHT_RESOLVERS) + } + + /** + * 鏍煎紡鍖栧皬鍖哄悕 + * @param name + * @return + */ + static String formatName(String name) { + name = name.replace("-鍟嗕綇", "").replace("-鍒", "") + //姹囩瀹跺洯骞冲畨閲岋紙杩侊級2015-2017 + name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "") + //鍥借锤澶╂偊涓�鏈�2017 + name = name.replaceAll("\\d\\d\\d\\d\$", "") + if (name ==~ /.+(\(|锛�)[\d涓�浜屼笁鍥涗簲鍏竷鍏節鍗佷笢鍗楄タ鍖梶\w]+(鏈焲鍖�)(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�)鍒(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�).+(\)|锛�)$/) { + name = name.replaceAll("(\\(|锛�).+(\\)|锛�)", '') + } else if (name.contains("路") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) { + println("---------------------------->" + name) + name = name.replaceAll("路|-", '') + println(name) + } + if (name ==~ /.{2,}璺�.{3,}$/ && !(name ==~ /.{2,}璺�.+鍙�/) && !(name ==~ /.{2,}璺�.+寮�/) && !(name ==~ /.*灏忓尯$/)) { + name = name.replaceAll("^.{2,}璺�", "") + } + return name + } + + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("涓�","1").replace("浜�","2").replace("涓�","3").replace("鍥�","4").replace("浜�","5").replace("涓�","2") + return resolver.getValue()(match) + } + } + return null + } + + /** + * 鍖归厤绗﹀悎鏉′欢鐨勫瓧绗︿覆 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("闈炴瘺鍧�")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|锛坾銆妡\"|鈥渱銆恷銆�).+?(\\)|銆媩锛墊\"|鈥潀銆憒銆�)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + static String parsePrice(String priceStr) { + def pattern =Pattern.compile("\\d+(\\.\\d+)?涓�") + String match = match(priceStr, pattern) + } + + static void main(String[] args) { + println parsePrice("椤剁娆e洯瑗胯嫅2瀹�1鍘�1鍗�492.3涓�") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy new file mode 100644 index 000000000..845ab47d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -0,0 +1,167 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class ZhugeFangDistCrawl { + + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new ZhugeFangDistCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/" + WebDriver driver = new ChromeDriver() + driver.manage().deleteAllCookies() + getRegions(driver, url) + } + + def getRegions(driver, url) { + for(def i=1;i<=11;i++){ + def newUrl = url+"page${i}/" + println newUrl + def doc = getDocument(newUrl, driver) + def total = parseList(doc) + +// def next = JQueryUtils.select("a:contains(下一页)", doc) +// if(next){ +// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() +// } + + } + + } + + def parseList(doc){ + def city= "福州" + def els = JQueryUtils.select("#listTableBox li", doc) + if (els) { + for(def e : els){ + def name = e.select(".house-name").text() + def str = e.select(".house-adr").text() + def address = null + if(str){ + if(str.contains("]")){ + address = str.substring(str.indexOf("]")+1).trim() + }else{ + address = str + } + } + def regionStr = null + if(str && str.contains("]") && str.contains("[")){ + regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]")) + } + + def region = null + def block = null + if(regionStr){ + def arrs = regionStr.split("-") + if(arrs && arrs.size()>0){ + if(arrs.size()==2){ + block = arrs[1].trim() + } + if(arrs.size()>=1){ + region = arrs[0].trim() + } + + } + } + def links = e.select(".house-name a") + def href = null + if(links){ + href = links[0].attr("href") + } + def priceStr = e.select(".price-number").text() + def price = null + if(priceStr && !priceStr.contains("暂无")){ + if(priceStr.contains("元")){ + priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim() + } + if(priceStr && priceStr.isNumber()){ + price = priceStr.toInteger() + } + } + def obj = findByParams(city, region, name) + if(obj){ + if(price){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price) + udata2.put("m_date", new Date()) + println "[upeate] ${region} "+name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + } + }else{ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + udata.put("block", block) + udata.put("url", href) + udata.put("address", address) + udata.put("name", name) + udata.put("avg_price", price) + udata.put("s_date", SHORT_DATE) + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("source", "诸葛找房") + thirdPartyColl.save(udata) + println "[save] ${region} "+name+" "+priceStr + } + } + } + return els.size() + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", "诸葛找房") + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } + + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(10000) + def els = JQueryUtils.select("#listTableBox li", doc) + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy new file mode 100644 index 000000000..47c9fe748 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy @@ -0,0 +1,123 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class DistrictMapMatch { + + static main(args) { + def task = new DistrictMapMatch() + task.mapMatch() + } + + + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d-> + if(!d['H']){ + index++ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// if (!map || !map.lat) { +// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); +// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// } + if(map){ + d['F'] = map.id + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + d['N'] = map.source + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 95bfde522..fe522570a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -945,20 +945,20 @@ class CZDataBatch extends BaseBatch{ if(price){ return price.avg_price; }else{ - //样例估价 - def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); - try{ - JSONObject jsonObject = JSONObject.fromObject(s); - def map = DataHandleUtils.JsonToMap(jsonObject); - if(map.code==200){ - if(map.result){ - return map.result.avgPrice; - } - - } - }catch(Exception e){ - - } +// //样例估价 +// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); +// try{ +// JSONObject jsonObject = JSONObject.fromObject(s); +// def map = DataHandleUtils.JsonToMap(jsonObject); +// if(map.code==200){ +// if(map.result){ +// return map.result.avgPrice; +// } +// +// } +// }catch(Exception e){ +// +// } //分期找总期 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 011f434e3..9a9fd90ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger import java.util.regex.Matcher -import java.util.regex.Patter +import java.util.regex.Pattern /** * 第三方数据标准化处理 @@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{ def filterSpacialChar(def s){ def pattern = ~/[()!;\·]\[【】()]/ def matcher = pattern.matcher(s) + def count = matcher.getCount() if(count && count>0){ return matcher.replaceAll('_') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index fe433f087..21887e3db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -169,7 +169,7 @@ public class CommonUtils { static void sendMessage(String queueName, byte[] bytes){ ConnectionFactory factory = new ConnectionFactory(); - factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f"); Connection conn = factory.newConnection(); Channel channel = conn.createChannel(); channel.queueDeclare(queueName, true, false, false, null); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f0186187b..3d4e855f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -348,6 +348,34 @@ public class OpenApiClient { } + public static JSONObject standard1Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap(); From 047467f6c29984e622c9d54363291bd9f27a7263 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 1 Jun 2022 13:47:06 +0800 Subject: [PATCH 81/83] update --- .../ccbevaluation/data/CcbDataStat.groovy | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy new file mode 100644 index 000000000..1aa5a391f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +class CcbDataStat { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new CcbDataStat() +// task.run() + task.tradeRun() + } + + def tradeRun(){ + def ccbCitys = CcbCommonUtils.citys; + def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd") + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d -> + if(d['A']){ + if(ccbCitys.contains(d['A'])){ + d['D'] = findOracleTotalCount(d['A']) + }else{ + def totalcount = findAllTradeCount(d['A'], startDate, endDate) + def repeatCount = findRepeatCount(d['A'], startDate, endDate) + def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate) + def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate) + d['B'] = totalcount + d['C'] = repeatCount + d['D'] = matchTotalCount + d['E'] = matchRepeatTotalCount + } + + } + }) + } + + + + def findRepeatCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + + def dbcursor = DEAL_PRICE_COLL.find(q) + def set = new HashSet() + for(def d : dbcursor){ + def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd") + if(!set.contains(key)){ + set << key + } + } + return set.size() + } + + def findMatchRepeatTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def set = new HashSet() + for(def d : dbcursor){ + def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd") + if(!set.contains(key)){ + set << key + } + } + return set.size() + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + public static def findAllTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) +// q.put("match_id", new BasicDBObject('$exists', false)) + def total = DEAL_PRICE_COLL.count(q) + return total + } + + def findOracleTotalCount(city){ + def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate + count = r.count + }) + return count + } + + def run(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d-> + if(d['A']) { + def citycode = cityCodeDataMap.get(d['A']) + def total = findTotalCount(citycode) + def f_count = findDistFinishDate(citycode) + def b_count = findBuildTypeCount(citycode) + def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total) + def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total) + d['C'] = f_pre + d['D'] = b_pre + d['E'] = total + println d['A'] +" "+ total +" "+f_count+" "+b_count + } + }) + } + + def findTotalCount(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findDistFinishDate(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findBuildTypeCount(citycode){ + def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } +} From efad3cb06d2a3b4dfc4a88d37c29b4c85b455644 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 1 Jun 2022 13:48:03 +0800 Subject: [PATCH 82/83] add --- .../script/gj/crawl/AnjukeHouseCrawl.groovy | 189 +++++++++++++ .../script/gj/crawl/DataCrawlUtils.groovy | 250 ++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy new file mode 100644 index 000000000..1773bdd8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -0,0 +1,189 @@ +package com.fangjia.script.gj.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeHouseCrawl { + def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" + static main(args) { + def task = new AnjukeHouseCrawl(); + task.run(); + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + for(def i=24;i<50;i++){ + def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation" + def doc = getDocument(url, driver) + def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) + println "页数:"+i+" 记录数:"+eles.size() + if(eles){ + parser(eles, driver) + } + } + } + + def parser(eles, driver){ + def index = 0 + for(def e : eles){ + index ++ + def href = e.attr("href") + def doc = getDocument(href, driver) +// parserHtml(doc) + def newUrl = href.substring(0, href.indexOf("?")) + sendMq(newUrl, doc) + + println "[ "+index +"] "+ newUrl + } + } + + def sendMq(href, doc){ + JSON json = new JSONObject() + json.put("project", "sell_anjuke") + json.put("url", href) + + JSON result = new JSONObject() + result.put("content", doc.toString()) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + } + + def parserHtml(doc) { + String city = null + def result = [:] + result.siteCode = "安居客" + if (doc) { + result.title = JQueryUtils.selectText([".banner-title h1"], doc) + def heads = JQueryUtils.select(".crumbs a", doc) + + if (heads && heads.size() >= 5) { + result.city = heads[0].text().replace("房产网", "") + def orgRegion = heads[2].text().replace("二手房", "") + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + result.block = heads[3].text().replace("二手房", "") + result.district_name = heads[4].text() + } else if (heads && heads.size() >= 3) { + result.city = heads[0].text().replace("房产网", "") + result.district_name = heads[2].text() + def eles = JQueryUtils.select("li:contains(所在位置:) a", doc) + if (eles && eles.size() > 0) { + def orgRegion = eles[0].text() + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + } + } + + if (!result.district_name) { + println " [未解析出(城市-区域-名称)信息]" + } + println result + + + def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc) + if (priceStr) { + if (priceStr.isNumber()) { + result.total_price = priceStr.toDouble() * 10000 + } + } + def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc) + if (roomInfoStr) { + Integer room = DataCrawlUtils.parseRoom(roomInfoStr) + Integer hall = DataCrawlUtils.parseHall(roomInfoStr) + Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr) + + if (room) { + result.room = room + } + if (hall) { + result.hall = hall + } + if (toilet) { + result.toilet = toilet + } + } + + def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "") + if (areaStr) { + result.area = areaStr.toDouble() + } + def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc) + result.direction = DataCrawlUtils.parseDirection(directionStr) + + def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc) + if (buildDateStr && buildDateStr.contains("/")) { + def arrs = buildDateStr.split("\\/") + if (arrs && arrs.size() == 2) { + result.build_date = arrs[0] + result.estate_type = arrs[1] + } + } + + result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc) + if (result.floor_info && result.floor_info.contains("共")) { + def height = result.floor_info + if (result.floor_info.contains("(")) { + height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")")) + } + height = height.replace("共", "").replace("层", "") + if (height) { + result.height = height.toInteger() + } + } + + + if (result.area && result.total_price) { + result.avg_price = (result.total_price / result.area).toInteger() + } + + def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc) + result.fitment = DataCrawlUtils.parseFitment(fitmitStr) + + + def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc) + + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(5000) + def els = JQueryUtils.select(".list-main .list-left .list", doc) + if(!els){ + els = JQueryUtils.select(".props-main", doc) + } + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy new file mode 100644 index 000000000..465ae5f04 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy @@ -0,0 +1,250 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataCrawlUtils { + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str -> + str = str.replaceAll("室|房", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str -> + str = str.replace("厅", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str -> + str = str.replace("卫", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str -> + str = str.replace("万", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str -> + str = str.replace("元/㎡", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str -> + str = str.replace("元/m?", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str -> + str = str.replace("元/平", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str -> + str = str.replace("元", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str -> + str = str.replace("平米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str -> + str = str.replace("平方米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str -> + str = str.replace("平", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str -> + str = str.replace("㎡", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str -> + str = str.replace("m?", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str -> + str = str.replace("共", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str -> + str = str.replace("总", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str -> + str = str.replace("第", "").replaceAll("层|楼", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + + } + + /** + * 解析室数 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2") + return resolver.getValue()(match) + } + } + return null + } + + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + +} From f63bfe70e13c87f65598e2288ebae530894fc3df Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 6 Jun 2022 18:04:58 +0800 Subject: [PATCH 83/83] =?UTF-8?q?=E5=B8=B8=E5=B7=9E=E8=B7=91=E6=89=B9?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/jiangnanblank/CZDataBatch.groovy | 63 +++++++++++++------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index fe522570a..926581423 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.AMQP import net.sf.json.JSONObject import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.bson.types.ObjectId @@ -46,11 +47,17 @@ class CZDataBatch extends BaseBatch{ @Override void process(Object args) { def citys = CommonDatautils.findAllCity(); + citys.remove("常州"); citys.add(0, "常州") for(def city : citys){ + + def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) + def commids = findAllJnBankDatas(city) + def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); - districtRun(districts); + districtRun(districts, statMap, commids); def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); @@ -117,7 +124,7 @@ class CZDataBatch extends BaseBatch{ return rMap; } - def districtRun(def districts){ + def districtRun(def districts, def statMap, commids){ def total = districts.size(); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(total) @@ -127,7 +134,7 @@ class CZDataBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - converDistrict(district); + converDistrict(district, statMap, commids); println totalCount.get()+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]"; } catch (Exception e) { e.printStackTrace() @@ -158,7 +165,7 @@ class CZDataBatch extends BaseBatch{ // executorService.awaitTermination(30, TimeUnit.MINUTES) } - def converDistrict(def dist){ + def converDistrict(def dist, def statMap, def commids){ def map = [:] //楼盘编号 /* def c_id = czIdMap.get(dist.city+"#"+dist.region+"#"+dist.name) @@ -179,7 +186,11 @@ class CZDataBatch extends BaseBatch{ map.estate_type2 = dist.estate_type2 map.build_structure = dist.build_structure // 楼盘均价 - map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block); + def priceObj = statMap.get(dist.region+"#"+dist.name) + if(priceObj && priceObj.avg_price){ + map.avg_price = priceObj.avg_price + } + //map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block); // 省 市 区 if(dist.city == "常州"){ map.province_code = regionMap.get("江苏"); @@ -190,23 +201,25 @@ class CZDataBatch extends BaseBatch{ map.city = dist.city; map.region = dist.region; - + if(!commids.contains(map.num_id)){ + // 公共服务配套 + map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2); + // 基础设施 + map.facilities_code = "01"; + // 居住社区成熟度 + map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2); + // 交通条件 + map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2); + // 泊车条件 / + map.stop_car_code = getStopCarCode(dist); + // /学区房类型 + map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ; + } // 地址 map.address = dist.address // 所属区域 map.in_region_code = getRegionCode(dist.region); - // 公共服务配套 - map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2); - // 基础设施 - map.facilities_code = "01"; - // 居住社区成熟度 - map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2); - // 交通条件 - map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2); - // 泊车条件 / - map.stop_car_code = getStopCarCode(dist); - // /学区房类型 - map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ; + //绿化率 map.virescence_percent = getVirescencePercent(dist.virescence_percent, map.build_date); //容积率 @@ -1102,4 +1115,18 @@ class CZDataBatch extends BaseBatch{ return list; } + def findAllJnBankDatas(city){ + def q = new BasicDBObject() + q.put("city", city) + + def keys = new BasicDBObject() + keys.put("num_id", 1) + def dbcursor = DIST_COLL.find(q, keys) + def set = new HashSet() + for(def d : dbcursor){ + set << d.num_id + } + return set; + } + }