From b8570b104e1209209880d6ed81d3e095a558cad2 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 11 May 2022 12:39:35 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9D=BF=E5=9D=97?= =?UTF-8?q?=E6=89=80=E5=B1=9E=E5=8C=BA=E5=9F=9F=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/geo/BlockRegion.groovy | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy new file mode 100644 index 000000000..bdd0d1110 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.geo + +import com.fangjia.script.gj.location.Point +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.xh.InsidePolygon +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017) + +collection.find(new BasicDBObject()).forEach({ obj -> + if (obj.type == 'block') { + if (obj.region instanceof String) { + return; + } + BasicDBList reigon = (BasicDBList)obj.region + int count = 0 + List info = new ArrayList<>() + for (String s : reigon) { + info.add(s) + } + if (info.size() > 1) { + boolean find = false + for (String s : info) { + if (find) { + break + } + collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 -> + String border = obj2.border + String points = border.replaceAll(",", "-").replaceAll(";", "-") + List pointList = InsidePolygon.createPoint(points); + com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat) + if (InsidePolygon.aboutZone(point, pointList)) { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", obj2.name))); + find = true; + } + }) + } + } else { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", info.get(0)))); + } + } +}) \ No newline at end of file From a0874980d9056863a5e73032f3a8cca17b0e872c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 12 May 2022 09:29:14 +0800 Subject: [PATCH 2/2] update --- .../chongqing/MatchEvaluateRegionTest.groovy | 25 +- .../script/chongqing/MatchEvaluateTest.groovy | 2 +- .../fangjia/script/chongqing/PocTest.groovy | 510 ++++++++++++++++++ .../DistrictAroundFacilitiesBatch.groovy | 2 +- .../script/gj/paimai/PaimaiTest.groovy | 87 ++- .../script/loudong/CqfdcChromeCrawl.groovy | 12 +- .../fangjia/script/loudong/CqfdcMatch.groovy | 432 ++++++++++++++- .../temp/shenjian/util/SeaweedUtils.groovy | 4 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 56 +- .../TestCrawlTotalCount.groovy | 4 +- .../fangjia/script/util/OpenApiClient.groovy | 171 ++++++ 11 files changed, 1249 insertions(+), 56 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy index bae011454..cd0df4d4c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -5,6 +5,7 @@ import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.paimai.DataUtil import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.CommonUtils @@ -22,16 +23,19 @@ class MatchEvaluateRegionTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def total = 1500 + static main(args) { def task = new MatchEvaluateRegionTest() task.run()//小区匹配 task.run1()//地图匹配 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 task.addPrice() + task.addArroundPrice() task.addAuthor() } def run(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) @@ -51,7 +55,6 @@ class MatchEvaluateRegionTest { } def run1(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['F']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) @@ -71,7 +74,6 @@ class MatchEvaluateRegionTest { } def addPrice(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['L']){ def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) @@ -91,9 +93,22 @@ class MatchEvaluateRegionTest { } + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A'] && !row['K']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['M'] = priceObj.unitPrice + row['N'] = (int) (priceObj.unitPrice * row['C']) + row['Q'] = "周边价格" + } + } + }); + } + def addAuthor(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" - def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def total = 747 def personTotal = authors.size() def avg = Math.ceil(total/personTotal) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy index e0a41a599..936b3b582 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -117,7 +117,7 @@ class MatchEvaluateTest { def addAuthor(){ // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" - def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def personTotal = authors.size() def avg = Math.ceil(total/personTotal) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy new file mode 100644 index 000000000..804d5891a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy @@ -0,0 +1,510 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.util.SourceNameUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CopyOnWriteArrayList + +class PocTest { + + static main(args) { + def task = new PocTest(); + task.run(); + } + + def run(){ +// match("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx") + +// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx") +// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx") +// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") +// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") + createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['P'] && row['C']){ + def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['P'] = jsonObject.get("name") + row['Q'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['Q']){ + def dist = CommonDatautils.findDistrictById(row['Q']) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['E'] = priceObj.avg_price + } + } + } + }) + } + + def standard(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + def jsonObject = OpenApiClient.standard2Action(row['A'], row['C']) + if(jsonObject){ + println jsonObject + row['F'] = jsonObject.get("province") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("township") + row['J'] = jsonObject.get("street") + row['K'] = jsonObject.get("streetNumber") + row['L'] = jsonObject.get("buildingNumber") + row['M'] = jsonObject.get("unitNumber") + row['N'] = jsonObject.get("floor") + row['O'] = jsonObject.get("roomNumber") + } + } + + }) + } + + def addLoudongNum(path){ + + ExcelUtils.multiThreadingModify(path, { row -> + if (row['Q'] && row['L']) { + def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + if (jsonObject && jsonObject.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['S'] = obj.id + row['R'] = obj.houseNumber + } + } + + if(row['O']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['U'] = obj.id + row['T'] = obj.roomNum + } + } + } + } + }) + } + + def createLoudong(path, output){ + def rets = new CopyOnWriteArrayList() + def datas = ExcelUtils.read(path, 0, 1) + rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"] + for(def row : datas){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + if (row['Q'] && row['L']) { + def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) { + def obj = jsonObject.result[0] + data << obj.houseNumber + data << obj.id + data << obj.type// 物业类型 + data << ""//主朝向 + data << obj.floors//地上层数 + data << ""//地下层数 + data << obj.buildYear;//竣工日期 + data << ""//梯 + data << ""//户 + data << obj.households//总户数 + data << ""//景观 + }else{ + data << "" + data << "" + data << ""// 物业类型 + data << ""//主朝向 + data << ""//地上层数 + data << ""//地下层数 + data << ""//竣工日期 + data << ""//梯 + data << ""//户 + data << ""//总户数 + data << ""//景观 + } + + if(row['O']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + + data << obj.roomNum//户号 + data << obj.id//户ID + data << obj.direction//主 + data << obj.sight//景观 + if(obj.roomNum && obj.roomNum.contains("跃")){ + data << "跃式"//户型结构 + }else{ + data << ""//户型结构 + } + + } + } + } + } + rets << data + } + ExcelUtils.write(output, rets, "底层数据-楼栋-户") + } + + def addDetail(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"] + for(def d : datas){ + if(d['Q']){ + def dist = CommonDatautils.findDistrictById(d['Q']) + if(dist){ + def row = [] + def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name) + if(jsonObject) { + + row << d['A'] + row << d['B'] + row << d['C'] + row << jsonObject.get("name")//小区名称 + row << d['Q']//小区ID + if(jsonObject.get("alias")){ + row << jsonObject.get("alias").join(",").replace('"',"")//小区别名 + }else{ + row << "" + } + + row << jsonObject.get("address")//小区地址 + row << jsonObject.get("commercialArea")//板块 + row << jsonObject.get("township")//街道*********** + row << jsonObject.get("avgPrice")//基准价 + row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度 + row << jsonObject.get("finishBuildTime")//竣工时间 + row << jsonObject.get("developer")//开发商名称 + row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率 + row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率 + row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数 + row << jsonObject.get("propertyManagement")//物业公司 + row << jsonObject.get("managementFee")//物业费 + row << jsonObject.get("buildingCount")//住宅幢数 + row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数 + row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比********* + row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积 + + } + + def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name) + if(facilityObj && facilityObj.result){ + def result = facilityObj.result + if(result.keySchool){ + row << '是' + }else { + row << "" + } + + if(result.school){ + row << result.school.join(",").replace('"',"") + }else { + row << "" + } + + def bus = [] + if(result.bus){ + bus.addAll(result.bus) + } + if(result.metro){ + bus.addAll(result.metro) + } + + if(bus){ + row << bus.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.supermarket){ + row << result.supermarket.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.hospital){ + row << result.hospital.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.bank){ + row << result.bank.join(",").replace('"',"") + }else{ + row << "" + } + } + rows << row + } + } + } + ExcelUtils.write(output, rows, "底层数据-小区") + } + + def createPaimai(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"] + def keys = [] + for(def d : datas){ + if(d['Q']) { + def dist = CommonDatautils.findDistrictById(d['Q']) + if (dist) { + def key = dist.city+"#"+dist.region+"#"+dist.name + if(keys.contains(key)){ + continue + } + keys << key + def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10) + if(jsonObject && jsonObject.result){ + def results = jsonObject.result + for(def r: results){ + def row = [] + row << d['A'] + row << d['B'] + row << d['C'] + row << r.matchName + row << r.address + row << r.houseNum + row << r.roomNum + row << r.area + row << r.startDate + row << r.count + row << r.status + row << r.tradeAvgPrice + row << r.tradePrice + row << r.id + rows << row + } + println dist.name +" "+results.size() + } + } + } + } + ExcelUtils.write(output, rows, "拍卖案例") + } + + def createHouse(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "挂牌" + row << r.districtName + if(r.estateType){ + row << r.estateType + }else{ + if(r.purpose){ + row << r.purpose + }else{ + row << "" + } + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.pDate + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + row << r.direction + row << SourceNameUtil.getResourceName(r.root) + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "挂牌案例") + } + + def createEvaluation(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "评估" + row << r.commName + row << "" + + if(r.roomcnt){ + def roomStr = r.roomcnt+"室" + if(r.livingroomcnt){ + roomStr = roomStr +r.livingroomcnt+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.buildArea + row << r.fitment + row << r.cDate.substring(0, r.cDate.indexOf("T")) + row << "" + row << r.unitPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.floorcnt){ + row << r.floorcnt + }else{ + row << "" + } + + if(r.houseaspect && !r.houseaspect.contains("暂")){ + row << r.houseaspect + }else{ + row << "" + } + + row << "房价网" + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "评估案例") + } + + + def createDealPrice(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "成交" + row << r.districtName + if(r.type){ + row << r.type + }else{ + row << "" + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.tradeDate.substring(0, r.tradeDate.indexOf("T")) + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + if(r.direction && !r.direction.contains("暂")){ + row << r.direction + }else{ + row << "" + } + + row << r.source + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "成交案例") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy index 2a2f5890f..63d7f841f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy @@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch { @Override public void process(Object args) { - def cities = ["北京", "上海"] + def cities = ["北京", "上海"]// for (def city : cities) { def ret = CommonDatautils.getAllDistricts("district", city) // def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index acced0959..c7451c5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON import com.fangjia.core.util.DataUtil import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import org.jsoup.Jsoup import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.ChromeDriver @@ -19,9 +22,91 @@ class PaimaiTest { DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { def task = new PaimaiTest(); - task.run(); +// task.run(); +// task.stat() +// task.updatePaimai() + task.updateFieldPaimai() + } + + def updateFieldPaimai(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) + def index = 0 + def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + for(def obj : dbcursor) { + index++ + println index + if(obj.depositPrice){ + obj.put("depositPrice", obj.depositPrice.toInteger()) + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + } + } + } + + def updatePaimai(){ + def q = new BasicDBObject() + def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def index =0 + for(def d : dbcursor){ + index++ + println index + def query = new BasicDBObject() + query.put("url", d.url) + def dbo = paimai43_COLL.findOne(query); + + DBObject obj = new BasicDBObject(); + d.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimai43_COLL.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + } + + def stat(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', ["重庆","重庆市"])) + def dbcursor = paimaiResultColl.find(q) + def keys = [] + for(def db : dbcursor){ + if(db.matchCity){ + def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName; + if(!keys.contains(key)){ + keys << key + } +// try{ +// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// db.matchCity = jsonObject.get("city") +// db.matchRegion = jsonObject.get("district") +// db.matchName = jsonObject.get("name") +// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db)) +// println db.address+" "+db.matchName +// } +// }catch(Exception e){ +// println "error "+db.address +// } + + } + } + println keys.size() } def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy index e5ffbd54f..172cc24b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.loudong +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -16,8 +17,15 @@ class CqfdcChromeCrawl { static main(args) { - def task = new CqfdcChromeCrawl(); - task.run() +// def task = new CqfdcChromeCrawl(); +// task.run() + + def name = "E3号楼" //69幢(规划C-7号楼) 规划5-5号楼 + def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + println s + + def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+单元") + println unitNumStr } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index bd59e65c7..9fbafb2ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -1,10 +1,14 @@ package com.fangjia.script.loudong +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject @@ -14,6 +18,8 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern class CqfdcMatch { @@ -23,6 +29,10 @@ class CqfdcMatch { static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl) + public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { @@ -33,6 +43,7 @@ class CqfdcMatch { // task.matchLoudong() task.addLoudong() +// task.addFdcLoudong() } def export(){ def q = new BasicDBObject() @@ -157,29 +168,378 @@ class CqfdcMatch { return rets; } + def addHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def keys = [] + def index = 0 + for(def d : dbcursor) { + index++ + println index + if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){ + continue + } + if(d.area.toDouble()<=25){ + continue + } + def newRegion = RegionData.formatRegion(d.city, d.region) + if (newRegion) { + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if (houseNumStr) { + house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "") + } + if (house_num) { + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if (unitNum) { + unitNum = unitNum.replace("单元", "") + } + def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num + if(!keys.contains(key) && d.room_num) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num) + if(!count){ + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("house_num_unit", unitNum) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "房谱网") + if(d.floor_heigth && d.floor_heigth.isNumber()){ + udata.put("house_height", d.floor_heigth.toInteger()) + } + if(d.floor && d.floor.isNumber()){ + udata.put("house_floor", d.floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", district_name) + q.put("house_num", house_num) + if(unitNum){ + q.put("unit_num", unitNum) + } + q.put("room_num", room_num) + def count = HOUSE_LOUDONG_LINE_COLL.count(q) + return count; + } + def findAllLoundongByHouseNum(){ def q = new BasicDBObject() - def dbcursor = loudongConn.find(q).batchSize(5000) - def rets = [] - def map = [:] + def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000) + def keys = [] for(def d : dbcursor){ - def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num - if(d.unit_num){ - key = key +"##"+d.unit_num - } - def detailMap= map.get(key) - if(!detailMap){ - detailMap = [:] - map.put(key,detailMap) - } + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if(unitNum){ + unitNum = unitNum.replace("单元","") + } + def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num + if(unitNum){ + key = key +"##"+unitNum + } + if(!keys.contains(key)){ + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num) + println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}" + if(obj){ + def udata = new BasicDBObject() + def areas = obj.areas + if(!areas.contains(d.area)){ + areas << d.area - if(detailMap.height){ + def forms = obj.from + if(!forms.contains("房谱网")){ + forms << "房谱网" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + }else{ + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("unit_num", unitNum) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("areas",[d.area]) + udata.put("from",["房谱网"]) + if(d.floor_heigth && d.floor_heigth.toString().toInteger()){ + udata.put("floors", d.floor_heigth.toString().toInteger()) + } + + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } } } - return rets; } + def addFdcHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000) + def keys = [] + def index = 0 + for(def d : dbcursor){ + index++ + println index + if(!d.use_type){ + continue + } + if(!d.use_type.contains("住宅")){ + continue + } + def newRegion = d.match_region + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num + if(!keys.contains(key)) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num) + if(!count){ + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def floor = null + if(d.room_num && d.room_num.contains("-")) { + def arrs = d.room_num.split("-") + floor = arrs[0] + } + Integer room = null + Integer hall = null + if(d.room_type && d.room_type.contains("室")){ + def roomStr = d.room_type.substring(0,d.room_type.indexOf("室")) + if(roomStr){ + roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + room = roomStr.toInteger() + } + + if(d.room_type.contains("厅")){ + def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅")) + if(hallStr){ + hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + hall = hallStr.toInteger() + } + } + } + + + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if(d.unit_num){ + udata.put("house_num_unit", d.unit_num) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "重庆网上房地产") + udata.put("house_height", maxHeight) + if(room){ + udata.put("room", room) + } + if(hall){ + udata.put("hall", hall) + } + if(floor && floor.isNumber()){ + udata.put("house_floor", floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + public String getMatchString(String text,String pattern){ + Pattern p=Pattern.compile(pattern); + Matcher m=p.matcher(text); + if(m.find()){ + return m.group(0) + } + return null; + } + + def addFdcBaseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + if(!d.match_region){ + continue + } + def newRegion = d.match_region + + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num + if(d.unit_num) { + key = key + "##" + d.unit_num + + if (!keys.contains(key)) { + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num) + println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}" + + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + if (obj) { + + def areas = obj.areas + if (!areas.contains(d.area)) { + areas << d.area + + def forms = obj.from + if (!forms.contains("重庆网上房地产")) { + forms << "重庆网上房地产" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + } else { + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if (d.unit_num) { + udata.put("unit_num", d.unit_num) + } + if (d.address) { + def address = AddressUtil.formatAddress(d.city, d.address) + if (address) { + udata.put("address", address) + } + } + udata.put("areas", [d.area]) + udata.put("from", ["重庆网上房地产"]) + udata.put("floors", maxHeight) + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findMaxHeight(city, name,house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("name",name) + q.put("house_num_desc",house_num) + q.put("unit_num",unit_num) + def dbcursor = loudongFdcConn.find(q) + def heights = [] + for(def h : dbcursor){ + if(h.nominal_floor && h.nominal_floor.isNumber()){ + def height = h.nominal_floor.toInteger() + if(!heights.contains(height)){ + heights << height + } + } + } + if(heights){ + heights = heights.sort() + return heights[heights.size()-1] + } + return null + } + + def findLoudongOnLine(city, region, name, house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + q.put("house_num",house_num) + if(unit_num){ + q.put("unit_num",unit_num) + } + def dbo = LOUDONG_LINE_COLL.findOne(q) + return dbo + } + + def stat(){ def q = new BasicDBObject() @@ -291,7 +651,51 @@ class CqfdcMatch { } def addLoudong(){ +// def stat = statBaseLoudong() +// println stat +// findAllLoundongByHouseNum(); + def stat1 = statHouseLoudong() + println stat1 +// addHouseLoudong() + } + def addFdcLoudong(){ +// addFdcBaseLoudong() + addFdcHouseLoudong() + } + + def statHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.district_id && !ids.contains(d.district_id)){ + ids << d.district_id + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + + def statBaseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){ + ids << d.id_in_seaweed + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 668172385..c17285a78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -128,10 +128,10 @@ class SeaweedUtils { */ static formatHouseholdCount(def household_count) { if (household_count && FieldValidatorUtils.checkValue(household_count) ) { - if (Long.valueOf(household_count) < 50) { + if (Integer.valueOf(household_count) < 50) { return "" // 户数 } - return household_count + return household_count.toInteger() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ee0977a28..c676385ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() - def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", @@ -192,37 +192,37 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] - for(def city : citys){ - def str = cityList.get(city) - for(def i=0;i<1;i++){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } - } - } - -// cityList.each{city, str-> +// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] +// for(def city : citys){ +// def str = cityList.get(city) // for(def i=0;i<1;i++){ -// if(citys.contains(city)){ -// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); -// if(districts){ -// def arrs = str.split("\\|") -// def url = "https://"+arrs[0]+".58.com/xiaoqu/" -// distMap.put(city, districts) -// -// addCrawlKeys(city); -// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); -// } +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); // } // } // } + cityList.each{city, str-> + for(def i=0;i<1;i++){ +// if(citys.contains(city)){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } +// } + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index bd0fbb3f3..4866f25b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202202); - def count2 = findTotalCount(source, 202203); + def count1 = findTotalCount(source, 202203); + def count2 = findTotalCount(source, 202204); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 5ef52da25..29c5f5f4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -229,6 +229,93 @@ public class OpenApiClient { return null } + public static JSONObject searchLoudong(String id, String houseNum, String unitNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("roomNumber", roomNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standard2Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap(); @@ -593,6 +680,90 @@ public class OpenApiClient { } + static JSONObject facilitys(String city, String region, String name) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("region", region) + params.put("name", name) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject searchPaimais(String city, String region, String name, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("district", region) + params.put("name", name) + params.put("limit", "10") + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject searchDealPrice(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject houseList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startTime", startDate) + params.put("pageSize", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject evaluaionList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + static JSONObject districtDetail(String city, String region, String name) { Map params = new HashMap(); params.put("token", TOKEN);