From 226afe9bc4b44663f6143f732d3612b2c3b7800d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Nov 2023 17:16:00 +0800 Subject: [PATCH] update --- .../BatchAddPersonGuiPrice.groovy | 5 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 1 - .../fangjia/script/gj/AddAvgpriceBatch.groovy | 15 +- .../com/fangjia/script/gj/OpenApiUtils.groovy | 22 ++ .../fangjia/script/gj/abc/SuzouPocTest.groovy | 153 +++++++++ .../script/gj/chongqin/PocTest2023.groovy | 317 ++++++++++++++++++ .../fangjia/script/gj/district/ApiTest.groovy | 126 ++++++- .../gj/district/DistrictFieldAddBatch.groovy | 20 +- .../gj/district/FangjiaClientUtils.groovy | 29 +- .../script/gj/meishubao/MsbPocTest.groovy | 2 +- .../script/gj/paimai/PaimaiTest.groovy | 40 ++- .../gj/xiamen/SuggestionDataSynch.groovy | 26 +- .../fangjia/script/util/OpenApiClient.groovy | 36 +- 13 files changed, 766 insertions(+), 26 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqin/PocTest2023.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy index 683c3048a..a69810dab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy @@ -23,7 +23,8 @@ class BatchAddPersonGuiPrice { } def run(){ - def path = "D:\\数据\\价格\\江苏剩下5城人工指导价录入.xlsx" + + def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx" def datas = ExcelUtils.read(path, 0, 1) for(def d : datas){ def row = findGuidingPrice(d['B'], d['C'], d['D']) @@ -31,7 +32,7 @@ class BatchAddPersonGuiPrice { def obj = CommonDatautils.findDistrictById(d['A']) save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2) //发重算均价命令 - addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-10','yyyy-MM-dd')) + addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date()) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 061b315af..8b444eaba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,6 @@ class BigdataForsaleBatch extends BaseBatch{ println "房源时间:"+startDate+" "+endDate def houseIdsfilter = findAllHouseIdsByCity() - citys = ["上海"] for(def city : citys){ index++ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 6ed286549..28fb38a92 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -27,7 +28,19 @@ class AddAvgpriceBatch { def task = new AddAvgpriceBatch(); // task.run1(); // task.test() - task.sendPriceCommand() +// task.sendPriceCommand() + task.sendNewDistPrice() + } + + def sendNewDistPrice(){ + DBObject q = new BasicDBObject(); + q.put("status",0) + q.put("cat", "district"); + q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-11-14","yyyy-MM-dd"))) + def dbsour = DbUtils.seaweed.find(q) + for(def d : dbsour){ + addPrice(d.city,d.region,d.name, d.estate_type2, DateUtils.parseFromTightByPattern('2023-07-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-19','yyyy-MM-dd')) + } } def sendPriceCommand(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index a31642613..be02065e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -583,6 +583,28 @@ class OpenApiUtils { return null; } + static def jsHouseCaseByAddress(address, area){ + address = address.replace(" ","").replace("%","") +// def url = "http://open.fangjia.com/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + def url = "http://localhost:8080/fangjia-open/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + if(area){ + url = url +"&strctureArea="+area + } +// println url + String content = new HttpRequest().get(url, "utf-8"); + def dto = [] + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + }else{ + println url + } + } + return null; + } + static def financialRating(city, address){ def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" String content = new HttpRequest().get(url, "utf-8"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy new file mode 100644 index 000000000..001f9cea6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy @@ -0,0 +1,153 @@ +package com.fangjia.script.gj.abc + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class SuzouPocTest { + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new SuzouPocTest() + task.run() + } + + def run(){ + def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx" + addressEvalute(path) +// addFjwTobPrice(path) +// addThirdPrice(path) +// margeExcel(path) + } + + + def addressEvalute(path){ + + ExcelUtils.modify(path,{d-> + + if(!d['O'] && d['H']){ + def city = d['D'] + def region = d['E'] + def address = d['G'] + def name= d['H'] + def area = d['I'] + def floor = null + if( d['J'] && d['J'].toString().isNumber()){ + floor = d['J'].toString().toDouble().toInteger().toString() + } + def height = null + if(d['K'] && d['K'].toString().isNumber()){ + height = d['K'].toString().toDouble().toInteger().toString() + } + def ret1 = OpenApiClient.addressEvaluation(city, region, address, name, area, floor, height) + if(ret1){ + d['O']= ret1.city + d['P'] = ret1.district + d['Q'] = ret1.name + d['R'] = ret1.totalPrice + d['S'] = ret1.avgPrice + } + + def ret2 = OpenApiClient.addressEvaluation(city, region, address, name, area, null, null) + if(ret2){ + d['T'] = ret2.totalPrice + d['U'] = ret2.avgPrice + } + } + }) + } + + + def addFjwTobPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['O']){ + def priceObj = CommonDatautils.findDistrictPrice(d['O'], d['P'], d['Q']) + if(priceObj){ + d['V'] = priceObj.avg_price + d['W'] = priceObj.avg_price_2b + } + } + }) + } + + def addThirdPrice(path) { + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify(path, { d -> + if (d['O'] && !d['V']) { + def dist = CommonDatautils.findDistrict(d['O'], d['P'], d['Q'], "district") + if (dist) { + + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + d['V'] = priceObj.avg_price + d['W'] = priceObj.avg_price_2b + } + + + def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310) + def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name) + if (priceMap) { + if (priceMap.containsKey("lianjia_esf_price")) { + d['X'] = priceMap.get("lianjia_esf_price") + } + + if (priceMap.containsKey("beike_esf_price")) { + d['Y'] = priceMap.get("beike_esf_price") + } + + if (priceMap.containsKey("anjuke_esf_price")) { + d['Z'] = priceMap.get("anjuke_esf_price") + } + } + def cityId = cityCodeDataMap.get(dist.city) + def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId) + if (map) { + d['AA'] = map.avg_price + } + def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01') + if (tradePrice) { + d['AB'] = tradePrice.toInteger() + } + } + } + }) + } + + def margeExcel(path){ + def datas = ExcelUtils.read("D:\\数据\\江苏农行\\(合并)江苏农匹配不上149条-.xlsx", 0, 1) + def map = [:] + for(def d : datas){ + map.put(d['G'], d['O']) + } + + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农1062条.xlsx",{d-> + def commId = map.get(d['G']) + def dist = CommonDatautils.findDistrictById(commId) + if(dist){ + d['O'] = dist.city + d['P'] = dist.region + d['Q'] = dist.name + + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + d['V'] = priceObj.avg_price + d['W'] = priceObj.avg_price_2b + } + + } + }) + + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqin/PocTest2023.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqin/PocTest2023.groovy new file mode 100644 index 000000000..27ea14198 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqin/PocTest2023.groovy @@ -0,0 +1,317 @@ +package com.fangjia.script.gj.chongqin + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.FangjiaClientUtils + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 重庆poc测试-2023版 + */ +class PocTest2023 { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + + static main(args) { + def task = new PocTest2023(); + task.run(); + } + + + def run(){ +// addEvaluationPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试2.xlsx") +// addressEvalute("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") +// structure("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") //九级地址拆分 +// addThirdPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") + +// baseInfo("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx") +// baseInfoByName("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx") + + + distScore("D:\\数据\\重庆农商行2023\\楼盘评级样本.xlsx") + } + + def addEvaluationPrice(path){ + ExcelUtils.multiThreadingModify(path,{ d-> + + if(d['E']){ + def city = d['B'] + def region = d['C'] + def address = d['E'] + def area = d['D'] + + //旧地址估值 + def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null) + if(ret1){ + d['N']= ret1.city + d['O'] = ret1.district + d['P'] = ret1.name + d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0) + d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0) + def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district") + if(dist){ + d['Q'] = dist._id.toString() + } + } + } + }) + } + + def addressEvalute(path){ + + ExcelUtils.multiThreadingModify(path,{ d-> + + if(!d['N'] && d['E']){ + def city = d['B'] + def region = d['C'] + def address = d['E'] + def area = d['D'] + + //旧地址估值 + def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null) + if(ret1){ + d['N']= ret1.city + d['O'] = ret1.district + d['P'] = ret1.name + d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0) + d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0) + def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district") + if(dist){ + d['Q'] = dist._id.toString() + } + } + + //新的匹配算法 + def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false) + if(jsonObject != null && jsonObject.credit>0) { + d['R'] = jsonObject.get("city") + d['S'] = jsonObject.get("district") + d['T'] = jsonObject.get("name") + d['U'] = jsonObject.get("id") + } + + if( d['Q'] && d['U'] && d['U']==d['Q']){ + d['V'] = "T" + }else{ + d['V'] = "F" + } + + + } + }) + } + + + def structure(inputfile){ + ExcelUtils.multiThreadingModify(inputfile,{ d-> + if(d['E']){ + def jsonObject = FangjiaClientUtils.newStructureByLevel9(d['B'], d['E']) + if(jsonObject != null && jsonObject.msgCode==200 && jsonObject.result) { + def ret = jsonObject.result + def str = "" + if(ret.township){ + str = ret.township + } + if(ret.street){ + str = str + ret.street + } + d['H'] = str + d['I'] = ret.name + d['J'] = ret.buildingNumber + d['K'] = ret.unitNumber + d['L'] = ret.floor + d['M'] = ret.roomNumber + } + + } + }) + } + +// def addAuthor(inputfile){ +// def authors = ["曹","张", "陈", "许", "金玉"] +// ExcelUtils.modify(inputfile, { d -> +// if(d['V'] && d['V'] == "F"){ +// for(def name : authors){ +// d['W'] = name +// } +// } +// }); +// } + + def baseInfo(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { d -> + if(d['E']){ + def city = d['B'] + def region = d['C'] + def address = d['E'] + def area = d['D'] + + println address + //旧地址估值 + def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null) + if(ret1){ + d['R']= ret1.city + d['S'] = ret1.district + d['F'] = ret1.name + def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district") + if(dist){ + d['T'] = dist._id.toString() + d['G'] = dist.estate_company + d['H'] = dist.developer + d['I'] = dist.cubage_percent + d['J'] = dist.virescence_percent + d['K'] = dist.all_area + d['L'] = dist.loudong_count + d['M'] = dist.household_count + d['N'] = dist.estate_charge + if(dist.open_date){ + d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd") + } + + if(dist.is_elevator){ + d['P'] = "有" + }else{ + d['P'] = "无" + } + + def newCity = RegionData.formatCity(city) + def obj = findPaiMai(newCity, address) + if(obj){ + d['Q'] = "是" + } + } + } + } + }) + } + + def findPaiMai(city, address){ + def q = new BasicDBObject() + q.put("city", city) + def or = new BasicDBList() + or.add(new BasicDBObject("title", new BasicDBObject('$regex', address))) + or.add(new BasicDBObject("address", new BasicDBObject('$regex', address))) + q.append('$or', or) + def obj = paimai43_COLL.findOne(q) + return obj + } + + def baseInfoByName(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { d -> + if(d['F']) { + def city = d['B'] + def address = d['E'] + def dist = CommonDatautils.findDistrict(d['R'], d['S'], d['F'], "district") + if (dist) { + d['T'] = dist._id.toString() + d['G'] = dist.estate_company + d['H'] = dist.developer + d['I'] = dist.cubage_percent + d['J'] = dist.virescence_percent + d['K'] = dist.all_area + d['L'] = dist.loudong_count + d['M'] = dist.household_count + d['N'] = dist.estate_charge + if (dist.open_date) { + d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd") + } + + if (dist.is_elevator) { + d['P'] = "有" + } else { + d['P'] = "无" + } + + def newCity = RegionData.formatCity(city) + def obj = findPaiMai(newCity, address) + if (obj) { + d['Q'] = "是" + } + } + } + }) + } + + + def distScore(inputfile) { + ExcelUtils.multiThreadingModify(inputfile, { d -> + if (d['F']) { + def obj = getDistrictRank(d['R'], d['S'], d['F']) + if(obj){ + def score = obj.city_rank*100/obj.city_rank_total + score = DataUtil.getRound(score) + d['G'] = 100-score + d['H'] = obj.city_rank + d['I'] = obj.city_rank_total + d['J'] = obj.avg_price + } + } + }) + } + + + def getDistrictRank(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name",name) + return DISTRICT_ANALY_DATA_COLL.findOne(q) + } + + + def addThirdPrice(path) { + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify(path, { d -> + if (d['P']) { + def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district") + if (dist) { + +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// d['V'] = priceObj.avg_price +// d['W'] = priceObj.avg_price_2b +// } + + def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310) + def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name) + if (priceMap) { + if (priceMap.containsKey("lianjia_esf_price")) { + d['X'] = priceMap.get("lianjia_esf_price") + } + + if (priceMap.containsKey("beike_esf_price")) { + d['Y'] = priceMap.get("beike_esf_price") + } + + if (priceMap.containsKey("anjuke_esf_price")) { + d['Z'] = priceMap.get("anjuke_esf_price") + } + } + def cityId = cityCodeDataMap.get(dist.city) + def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId) + if (map) { + d['AA'] = map.avg_price + } + def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01') + if (tradePrice) { + d['AB'] = tradePrice.toInteger() + } + } + } + }) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy index f1336faa5..f5a50f0c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy @@ -27,10 +27,53 @@ class ApiTest { // task.similarRunTest()//相似房源接口测试 // task.loupanLevelTest()//楼盘评级接口测试 -// task.jsAbcHouseCastTest()//江苏农总行房源接口测试 - task.arroundFacilities()//周边poi +// task.jsAbcHouseCastTest()//北交所房源接口测试 小区 +// task.arroundFacilities()//北交所周边poi 小区 + + +// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价 +// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值 +// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址 + task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx") } + def addressEvaluate(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['B']){ + def ret1 = OpenApiClient.addressEvaluation(d['A'], null, d['B'], null, d['C'].toString().toDouble().toString(), null, null) + if(ret1){ + d['G']= ret1.city + d['H'] = ret1.district + d['I'] = ret1.name + d['J'] = ret1.totalPrice + d['K'] = ret1.avgPrice + } + } + }) + } + + def addressAvgPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['B']){ + println "[match] city:"+d['A']+" address:"+d['B'] + def jsonObject = FangjiaClientUtils.newMatchPrice(d['A'], d['B'], "property", false) + + if(jsonObject && jsonObject.containsKey("result")) { + def resultMap = jsonObject.get("result") + d['G'] = resultMap.get("city") + d['H'] = resultMap.get("district") + d['I'] = resultMap.get("name") + def type = resultMap.get("propertyType") + if(type && type!='null'){ + d['J'] = type + } + if(resultMap.get("avgPrice")){ + d['K'] = resultMap.get("avgPrice") + } + } + } + }) + } def loupanLevelTest(){ @@ -215,6 +258,7 @@ class ApiTest { row << d.name def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name) if(facilityObj && facilityObj.result){ + def result = facilityObj.result if(result.primarySchool){ row < + if(d['B']){ + def facilityObj = OpenApiClient.facilitysByAddress(d['A']+d['B']) + if(facilityObj && facilityObj.result){ + def result = facilityObj.result + d['G'] = result.city + d['H'] = result.region + d['I'] = result.districtName + if(result.primarySchool){ + d['J'] = result.primarySchool.join(",").replace('"',"") + }else { + d['J'] = "" + } + + if(result.school){ + d['K'] = result.school.join(",").replace('"',"") + }else { + d['K'] = "" + } + + if(result.bus){ + d['L'] = result.bus.join(",").replace('"',"") + }else{ + d['L'] = "" + } + + + if(result.metro){ + d['M'] = result.metro.join(",").replace('"',"") + }else{ + d['M'] = "" + } + + if(result.bank){ + d['N'] = result.bank.join(",").replace('"',"") + }else{ + d['N'] = "" + } + + + if(result.supermarket){ + d['O'] = result.supermarket.join(",").replace('"',"") + }else{ + d['O'] = "" + } + + if(result.hospital){ + d['P'] = result.hospital.join(",").replace('"',"") + }else{ + d['P'] = "" + } + } + } + }) + } + + def jsAbcHouseCastTestByAddress(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['B'] && !d['H']){ + def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCaseByAddress(d['A']+d['B'], d['C']) + if(results){ + d['H'] = results.minUnitPrice + d['I'] = results.maxUnitPrice + + def datas = results.datas + if(datas){ + def str = "" + for(def r : datas){ + str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n" + } + d['J'] = str + } + } + } + }); + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 400e81699..76fa40fb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -39,7 +39,7 @@ class DistrictFieldAddBatch extends BaseBatch{ //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" // - def sourceMap = ["安居客":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", + def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 @@ -47,14 +47,14 @@ class DistrictFieldAddBatch extends BaseBatch{ //"alias":"alias", //ownership_type:权属类型 为商品房、经适房、动迁房等 //house_property 产权性质:国有产权 "house_property":"网站没有", -// def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", -// "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", -// "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", -// "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", -// "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段 + def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", + "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", + "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", + "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", + "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段 - def fieldsMap = ["household_count":"households"] +// def fieldsMap = ["household_count":"households"] def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商", "virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型", @@ -72,7 +72,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity().reverse() + def citys = CommonDatautils.findAllCity() def analysRets = [] @@ -98,7 +98,7 @@ class DistrictFieldAddBatch extends BaseBatch{ } } println analysRets.toString() - sendStatEmail(headers, analysRets, rh_analysRets) +// sendStatEmail(headers, analysRets, rh_analysRets) // ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") } @@ -225,7 +225,7 @@ class DistrictFieldAddBatch extends BaseBatch{ } } - ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource) +// ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource) return fieldValueMap; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 8fc8f23cb..ab0877e12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -226,9 +226,9 @@ class FangjiaClientUtils { def jsonObj = null def content = null try { - address = address.replace("{","") - // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") - def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" + address = address.replace("{","").replace("\\","") + def url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" + // def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") @@ -253,7 +253,7 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://localhost:8080/fangjia-open/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" + def url = "http://open.fangjia.com/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") @@ -272,6 +272,27 @@ class FangjiaClientUtils { } + public static def newStructureByLevel9(city,address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","").replace("\\","") + def url = "http://192.168.0.100:32080/FangJiaAddressService/v2/structure?address="+URLEncoder.encode(address,"UTF-8") + + if(city){ + url = url + "&city="+URLEncoder.encode(city,"UTF-8") + } + + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + + public static def newStandard(address){ def jsonObj = null def content = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index 9fd731ffd..5f0c6fe19 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -16,7 +16,7 @@ class MsbPocTest { // def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx" def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx // standard(inputfile) -// match(inputfile) + match(inputfile) // matchOld(inputfile) // addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx") // test("D:\\数据\\美术宝\\test2.xlsx") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index 9a8e0c117..52add2e69 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -3,7 +3,10 @@ package com.fangjia.script.gj.paimai import com.alibaba.fastjson.JSON import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -35,7 +38,8 @@ class PaimaiTest { // task.updateFieldPaimai() // task.statCity() // task.jsStat() - task.selectData() +// task.selectData() + task.randomExtractData() } def selectData(){ @@ -768,4 +772,38 @@ class PaimaiTest { } } } + + def randomExtractData(){ + def runedcity = ["广州","重庆","南京","佛山","东莞","扬州","镇江","南通","惠州","珠海","连云港","淮安","清远","云浮","上海","深圳","苏州","徐州","盐城","宿迁","肇庆","韶关","汕尾","常州","湛江","泰州","无锡","江门","汕头","中山","合肥","河源","阳江","梅州","茂名","揭阳","福州","潮州"] + def citys = CommonDatautils.findAllCity() + def datas = [] + datas << ["城市", "标地物", "面积", "URL", "结束时间", "成交价"] + for(def c : citys){ + if(runedcity.contains(c)){ + continue + } + def q = new BasicDBObject() + q.put("city", c) + q.put("status", "已成交") + q.put("estateType","住宅") + q.put("area", new BasicDBObject('$exists', true)) + q.put("title",new BasicDBObject('$exists', true) ) + def dbcursor = paimai43_COLL.find(q).sort(new BasicDBObject("finishDate",-1)).limit(100) + def rows = [] + for(def d : dbcursor){ + if(d.title && !d.title.contains("商") && !d.title.contains("铺") && !d.title.contains("车位")){ + def date = DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") + rows << [d.city, d.title, d.area, d.url, date, d.tradePrice] + } + if(rows.size()>=40){ + break; + } + } + + datas.addAll(rows) + println c +" "+rows.size() + } + + ExcelUtils.write("D:\\数据\\北交所\\拍卖案例-住宅-测试.xlsx", datas, "数据1") + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 544d54541..c25d2360a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -9,6 +9,8 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ProvinceData import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import java.util.concurrent.* import java.util.concurrent.atomic.AtomicLong @@ -31,7 +33,6 @@ class SuggestionDataSynch extends BaseBatch{ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") - public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) def keys = new CopyOnWriteArraySet(); def fullCityMap = null @@ -43,7 +44,8 @@ class SuggestionDataSynch extends BaseBatch{ @Override public void process(Object args) { - run() +// run() + updateCode() } @@ -283,4 +285,24 @@ class SuggestionDataSynch extends BaseBatch{ } return retMap; } + + def updateCode(){ + def q = new BasicDBObject() + def dbcursor = administrative_division_100_CONN.find(q) + for(def d : dbcursor){ + def code = null + if(d.mca_district_code){ + code = d.mca_district_code + }else if(d.mca_city_code){ + code = d.mca_city_code + }else if(d.mca_province_code){ + code = d.mca_province_code + } + + + DBObject udata = new BasicDBObject() + udata.put("code", code); + administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)); + } + } } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 07bbbf6f4..a9c8b0b4c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -774,6 +774,22 @@ public class OpenApiClient { return null } + static JSONObject facilitysByAddress(address) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("address", address) + try { + String content = new HttpRequest().get("http://open.fangjia.com/district/addrFacilitys", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) { Map params = new HashMap(); params.put("token", TOKEN); @@ -1104,15 +1120,31 @@ public class OpenApiClient { return null } - public static JSONObject addressEvaluation(city, region, address, size){ + public static JSONObject addressEvaluation(city, region, address, name, size, floor, height){ try { HttpRequest request = new HttpRequest(); Map params = new HashMap(); params.put("city", city); - params.put("district", region); params.put("address", address); + if(region){ + params.put("district", region); + } + + if(name){ + params.put("name", name); + } + params.put("size", String.valueOf(size)); params.put("token", TOKEN); + params.put("__refresh_cache", "1") + if(floor){ + params.put("floor", floor) + } + + if(height){ + params.put("maxFloor", height) + } + String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8"); JSONObject result = JSONObject.fromObject(response); if (result.getInt("code") == 200) {