This commit is contained in:
2023-11-17 17:16:00 +08:00
parent e2a260b1a4
commit 226afe9bc4
13 changed files with 766 additions and 26 deletions
@@ -23,7 +23,8 @@ class BatchAddPersonGuiPrice {
}
def run(){
def path = "D:\\数据\\价格\\江苏剩下5城人工指导价录入.xlsx"
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx"
def datas = ExcelUtils.read(path, 0, 1)
for(def d : datas){
def row = findGuidingPrice(d['B'], d['C'], d['D'])
@@ -31,7 +32,7 @@ class BatchAddPersonGuiPrice {
def obj = CommonDatautils.findDistrictById(d['A'])
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
//发重算均价命令
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-10','yyyy-MM-dd'))
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date())
}
}
}
@@ -91,7 +91,6 @@ class BigdataForsaleBatch extends BaseBatch{
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
citys = ["上海"]
for(def city : citys){
index++
@@ -3,6 +3,7 @@ package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -27,7 +28,19 @@ class AddAvgpriceBatch {
def task = new AddAvgpriceBatch();
// task.run1();
// task.test()
task.sendPriceCommand()
// task.sendPriceCommand()
task.sendNewDistPrice()
}
def sendNewDistPrice(){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", "district");
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-11-14","yyyy-MM-dd")))
def dbsour = DbUtils.seaweed.find(q)
for(def d : dbsour){
addPrice(d.city,d.region,d.name, d.estate_type2, DateUtils.parseFromTightByPattern('2023-07-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-19','yyyy-MM-dd'))
}
}
def sendPriceCommand(){
@@ -583,6 +583,28 @@ class OpenApiUtils {
return null;
}
static def jsHouseCaseByAddress(address, area){
address = address.replace(" ","").replace("%","")
// def url = "http://open.fangjia.com/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
def url = "http://localhost:8080/fangjia-open/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(area){
url = url +"&strctureArea="+area
}
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
@@ -0,0 +1,153 @@
package com.fangjia.script.gj.abc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class SuzouPocTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new SuzouPocTest()
task.run()
}
def run(){
def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx"
addressEvalute(path)
// addFjwTobPrice(path)
// addThirdPrice(path)
// margeExcel(path)
}
def addressEvalute(path){
ExcelUtils.modify(path,{d->
if(!d['O'] && d['H']){
def city = d['D']
def region = d['E']
def address = d['G']
def name= d['H']
def area = d['I']
def floor = null
if( d['J'] && d['J'].toString().isNumber()){
floor = d['J'].toString().toDouble().toInteger().toString()
}
def height = null
if(d['K'] && d['K'].toString().isNumber()){
height = d['K'].toString().toDouble().toInteger().toString()
}
def ret1 = OpenApiClient.addressEvaluation(city, region, address, name, area, floor, height)
if(ret1){
d['O']= ret1.city
d['P'] = ret1.district
d['Q'] = ret1.name
d['R'] = ret1.totalPrice
d['S'] = ret1.avgPrice
}
def ret2 = OpenApiClient.addressEvaluation(city, region, address, name, area, null, null)
if(ret2){
d['T'] = ret2.totalPrice
d['U'] = ret2.avgPrice
}
}
})
}
def addFjwTobPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['O']){
def priceObj = CommonDatautils.findDistrictPrice(d['O'], d['P'], d['Q'])
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
}
})
}
def addThirdPrice(path) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { d ->
if (d['O'] && !d['V']) {
def dist = CommonDatautils.findDistrict(d['O'], d['P'], d['Q'], "district")
if (dist) {
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
if (priceMap) {
if (priceMap.containsKey("lianjia_esf_price")) {
d['X'] = priceMap.get("lianjia_esf_price")
}
if (priceMap.containsKey("beike_esf_price")) {
d['Y'] = priceMap.get("beike_esf_price")
}
if (priceMap.containsKey("anjuke_esf_price")) {
d['Z'] = priceMap.get("anjuke_esf_price")
}
}
def cityId = cityCodeDataMap.get(dist.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if (map) {
d['AA'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
if (tradePrice) {
d['AB'] = tradePrice.toInteger()
}
}
}
})
}
def margeExcel(path){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\(合并)江苏农匹配不上149条-.xlsx", 0, 1)
def map = [:]
for(def d : datas){
map.put(d['G'], d['O'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农1062条.xlsx",{d->
def commId = map.get(d['G'])
def dist = CommonDatautils.findDistrictById(commId)
if(dist){
d['O'] = dist.city
d['P'] = dist.region
d['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
}
})
}
}
@@ -0,0 +1,317 @@
package com.fangjia.script.gj.chongqin
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 重庆poc测试-2023版
*/
class PocTest2023 {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
static main(args) {
def task = new PocTest2023();
task.run();
}
def run(){
// addEvaluationPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试2.xlsx")
// addressEvalute("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
// structure("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") //九级地址拆分
// addThirdPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
// baseInfo("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
// baseInfoByName("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
distScore("D:\\数据\\重庆农商行2023\\楼盘评级样本.xlsx")
}
def addEvaluationPrice(path){
ExcelUtils.multiThreadingModify(path,{ d->
if(d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['N']= ret1.city
d['O'] = ret1.district
d['P'] = ret1.name
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['Q'] = dist._id.toString()
}
}
}
})
}
def addressEvalute(path){
ExcelUtils.multiThreadingModify(path,{ d->
if(!d['N'] && d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['N']= ret1.city
d['O'] = ret1.district
d['P'] = ret1.name
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['Q'] = dist._id.toString()
}
}
//新的匹配算法
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null && jsonObject.credit>0) {
d['R'] = jsonObject.get("city")
d['S'] = jsonObject.get("district")
d['T'] = jsonObject.get("name")
d['U'] = jsonObject.get("id")
}
if( d['Q'] && d['U'] && d['U']==d['Q']){
d['V'] = "T"
}else{
d['V'] = "F"
}
}
})
}
def structure(inputfile){
ExcelUtils.multiThreadingModify(inputfile,{ d->
if(d['E']){
def jsonObject = FangjiaClientUtils.newStructureByLevel9(d['B'], d['E'])
if(jsonObject != null && jsonObject.msgCode==200 && jsonObject.result) {
def ret = jsonObject.result
def str = ""
if(ret.township){
str = ret.township
}
if(ret.street){
str = str + ret.street
}
d['H'] = str
d['I'] = ret.name
d['J'] = ret.buildingNumber
d['K'] = ret.unitNumber
d['L'] = ret.floor
d['M'] = ret.roomNumber
}
}
})
}
// def addAuthor(inputfile){
// def authors = ["曹","张", "陈", "许", "金玉"]
// ExcelUtils.modify(inputfile, { d ->
// if(d['V'] && d['V'] == "F"){
// for(def name : authors){
// d['W'] = name
// }
// }
// });
// }
def baseInfo(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { d ->
if(d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
println address
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['R']= ret1.city
d['S'] = ret1.district
d['F'] = ret1.name
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['T'] = dist._id.toString()
d['G'] = dist.estate_company
d['H'] = dist.developer
d['I'] = dist.cubage_percent
d['J'] = dist.virescence_percent
d['K'] = dist.all_area
d['L'] = dist.loudong_count
d['M'] = dist.household_count
d['N'] = dist.estate_charge
if(dist.open_date){
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
}
if(dist.is_elevator){
d['P'] = "有"
}else{
d['P'] = "无"
}
def newCity = RegionData.formatCity(city)
def obj = findPaiMai(newCity, address)
if(obj){
d['Q'] = "是"
}
}
}
}
})
}
def findPaiMai(city, address){
def q = new BasicDBObject()
q.put("city", city)
def or = new BasicDBList()
or.add(new BasicDBObject("title", new BasicDBObject('$regex', address)))
or.add(new BasicDBObject("address", new BasicDBObject('$regex', address)))
q.append('$or', or)
def obj = paimai43_COLL.findOne(q)
return obj
}
def baseInfoByName(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { d ->
if(d['F']) {
def city = d['B']
def address = d['E']
def dist = CommonDatautils.findDistrict(d['R'], d['S'], d['F'], "district")
if (dist) {
d['T'] = dist._id.toString()
d['G'] = dist.estate_company
d['H'] = dist.developer
d['I'] = dist.cubage_percent
d['J'] = dist.virescence_percent
d['K'] = dist.all_area
d['L'] = dist.loudong_count
d['M'] = dist.household_count
d['N'] = dist.estate_charge
if (dist.open_date) {
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
}
if (dist.is_elevator) {
d['P'] = "有"
} else {
d['P'] = "无"
}
def newCity = RegionData.formatCity(city)
def obj = findPaiMai(newCity, address)
if (obj) {
d['Q'] = "是"
}
}
}
})
}
def distScore(inputfile) {
ExcelUtils.multiThreadingModify(inputfile, { d ->
if (d['F']) {
def obj = getDistrictRank(d['R'], d['S'], d['F'])
if(obj){
def score = obj.city_rank*100/obj.city_rank_total
score = DataUtil.getRound(score)
d['G'] = 100-score
d['H'] = obj.city_rank
d['I'] = obj.city_rank_total
d['J'] = obj.avg_price
}
}
})
}
def getDistrictRank(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name",name)
return DISTRICT_ANALY_DATA_COLL.findOne(q)
}
def addThirdPrice(path) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { d ->
if (d['P']) {
def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
if (dist) {
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// d['V'] = priceObj.avg_price
// d['W'] = priceObj.avg_price_2b
// }
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
if (priceMap) {
if (priceMap.containsKey("lianjia_esf_price")) {
d['X'] = priceMap.get("lianjia_esf_price")
}
if (priceMap.containsKey("beike_esf_price")) {
d['Y'] = priceMap.get("beike_esf_price")
}
if (priceMap.containsKey("anjuke_esf_price")) {
d['Z'] = priceMap.get("anjuke_esf_price")
}
}
def cityId = cityCodeDataMap.get(dist.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if (map) {
d['AA'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
if (tradePrice) {
d['AB'] = tradePrice.toInteger()
}
}
}
})
}
}
@@ -27,10 +27,53 @@ class ApiTest {
// task.similarRunTest()//相似房源接口测试
// task.loupanLevelTest()//楼盘评级接口测试
// task.jsAbcHouseCastTest()//江苏农总行房源接口测试
task.arroundFacilities()//周边poi
// task.jsAbcHouseCastTest()//北交所房源接口测试 小区
// task.arroundFacilities()//北交所周边poi 小区
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
}
def addressEvaluate(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def ret1 = OpenApiClient.addressEvaluation(d['A'], null, d['B'], null, d['C'].toString().toDouble().toString(), null, null)
if(ret1){
d['G']= ret1.city
d['H'] = ret1.district
d['I'] = ret1.name
d['J'] = ret1.totalPrice
d['K'] = ret1.avgPrice
}
}
})
}
def addressAvgPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
println "[match] city:"+d['A']+" address:"+d['B']
def jsonObject = FangjiaClientUtils.newMatchPrice(d['A'], d['B'], "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
d['G'] = resultMap.get("city")
d['H'] = resultMap.get("district")
d['I'] = resultMap.get("name")
def type = resultMap.get("propertyType")
if(type && type!='null'){
d['J'] = type
}
if(resultMap.get("avgPrice")){
d['K'] = resultMap.get("avgPrice")
}
}
}
})
}
def loupanLevelTest(){
@@ -215,6 +258,7 @@ class ApiTest {
row << d.name
def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.primarySchool){
row <<result.primarySchool.join(",").replace('"',"")
@@ -275,5 +319,83 @@ class ApiTest {
}
def arroundFacilitiesByAddress(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def facilityObj = OpenApiClient.facilitysByAddress(d['A']+d['B'])
if(facilityObj && facilityObj.result){
def result = facilityObj.result
d['G'] = result.city
d['H'] = result.region
d['I'] = result.districtName
if(result.primarySchool){
d['J'] = result.primarySchool.join(",").replace('"',"")
}else {
d['J'] = ""
}
if(result.school){
d['K'] = result.school.join(",").replace('"',"")
}else {
d['K'] = ""
}
if(result.bus){
d['L'] = result.bus.join(",").replace('"',"")
}else{
d['L'] = ""
}
if(result.metro){
d['M'] = result.metro.join(",").replace('"',"")
}else{
d['M'] = ""
}
if(result.bank){
d['N'] = result.bank.join(",").replace('"',"")
}else{
d['N'] = ""
}
if(result.supermarket){
d['O'] = result.supermarket.join(",").replace('"',"")
}else{
d['O'] = ""
}
if(result.hospital){
d['P'] = result.hospital.join(",").replace('"',"")
}else{
d['P'] = ""
}
}
}
})
}
def jsAbcHouseCastTestByAddress(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B'] && !d['H']){
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCaseByAddress(d['A']+d['B'], d['C'])
if(results){
d['H'] = results.minUnitPrice
d['I'] = results.maxUnitPrice
def datas = results.datas
if(datas){
def str = ""
for(def r : datas){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
}
d['J'] = str
}
}
}
});
}
}
@@ -39,7 +39,7 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["安居客":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
@@ -47,14 +47,14 @@ class DistrictFieldAddBatch extends BaseBatch{
//"alias":"alias",
//ownership_type:权属类型 为商品房、经适房、动迁房等
//house_property 产权性质:国有产权 "house_property":"网站没有",
// def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
// "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
// "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
// "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
// "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
"loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
def fieldsMap = ["household_count":"households"]
// def fieldsMap = ["household_count":"households"]
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
@@ -72,7 +72,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
def citys = CommonDatautils.findAllCity()
def analysRets = []
@@ -98,7 +98,7 @@ class DistrictFieldAddBatch extends BaseBatch{
}
}
println analysRets.toString()
sendStatEmail(headers, analysRets, rh_analysRets)
// sendStatEmail(headers, analysRets, rh_analysRets)
// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
}
@@ -225,7 +225,7 @@ class DistrictFieldAddBatch extends BaseBatch{
}
}
ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
// ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
return fieldValueMap;
}
@@ -226,9 +226,9 @@ class FangjiaClientUtils {
def jsonObj = null
def content = null
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
address = address.replace("{","").replace("\\","")
def url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
@@ -253,7 +253,7 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://localhost:8080/fangjia-open/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
def url = "http://open.fangjia.com/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
@@ -272,6 +272,27 @@ class FangjiaClientUtils {
}
public static def newStructureByLevel9(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","").replace("\\","")
def url = "http://192.168.0.100:32080/FangJiaAddressService/v2/structure?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newStandard(address){
def jsonObj = null
def content = null
@@ -16,7 +16,7 @@ class MsbPocTest {
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
match(inputfile)
// matchOld(inputfile)
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
// test("D:\\数据\\美术宝\\test2.xlsx")
@@ -3,7 +3,10 @@ package com.fangjia.script.gj.paimai
import com.alibaba.fastjson.JSON
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -35,7 +38,8 @@ class PaimaiTest {
// task.updateFieldPaimai()
// task.statCity()
// task.jsStat()
task.selectData()
// task.selectData()
task.randomExtractData()
}
def selectData(){
@@ -768,4 +772,38 @@ class PaimaiTest {
}
}
}
def randomExtractData(){
def runedcity = ["广州","重庆","南京","佛山","东莞","扬州","镇江","南通","惠州","珠海","连云港","淮安","清远","云浮","上海","深圳","苏州","徐州","盐城","宿迁","肇庆","韶关","汕尾","常州","湛江","泰州","无锡","江门","汕头","中山","合肥","河源","阳江","梅州","茂名","揭阳","福州","潮州"]
def citys = CommonDatautils.findAllCity()
def datas = []
datas << ["城市", "标地物", "面积", "URL", "结束时间", "成交价"]
for(def c : citys){
if(runedcity.contains(c)){
continue
}
def q = new BasicDBObject()
q.put("city", c)
q.put("status", "已成交")
q.put("estateType","住宅")
q.put("area", new BasicDBObject('$exists', true))
q.put("title",new BasicDBObject('$exists', true) )
def dbcursor = paimai43_COLL.find(q).sort(new BasicDBObject("finishDate",-1)).limit(100)
def rows = []
for(def d : dbcursor){
if(d.title && !d.title.contains("商") && !d.title.contains("铺") && !d.title.contains("车位")){
def date = DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
rows << [d.city, d.title, d.area, d.url, date, d.tradePrice]
}
if(rows.size()>=40){
break;
}
}
datas.addAll(rows)
println c +" "+rows.size()
}
ExcelUtils.write("D:\\数据\\北交所\\拍卖案例-住宅-测试.xlsx", datas, "数据1")
}
}
@@ -9,6 +9,8 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.*
import java.util.concurrent.atomic.AtomicLong
@@ -31,7 +33,6 @@ class SuggestionDataSynch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
def keys = new CopyOnWriteArraySet();
def fullCityMap = null
@@ -43,7 +44,8 @@ class SuggestionDataSynch extends BaseBatch{
@Override
public void process(Object args) {
run()
// run()
updateCode()
}
@@ -283,4 +285,24 @@ class SuggestionDataSynch extends BaseBatch{
}
return retMap;
}
def updateCode(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
for(def d : dbcursor){
def code = null
if(d.mca_district_code){
code = d.mca_district_code
}else if(d.mca_city_code){
code = d.mca_city_code
}else if(d.mca_province_code){
code = d.mca_province_code
}
DBObject udata = new BasicDBObject()
udata.put("code", code);
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
}
}
}
@@ -774,6 +774,22 @@ public class OpenApiClient {
return null
}
static JSONObject facilitysByAddress(address) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("address", address)
try {
String content = new HttpRequest().get("http://open.fangjia.com/district/addrFacilitys", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
@@ -1104,15 +1120,31 @@ public class OpenApiClient {
return null
}
public static JSONObject addressEvaluation(city, region, address, size){
public static JSONObject addressEvaluation(city, region, address, name, size, floor, height){
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("address", address);
if(region){
params.put("district", region);
}
if(name){
params.put("name", name);
}
params.put("size", String.valueOf(size));
params.put("token", TOKEN);
params.put("__refresh_cache", "1")
if(floor){
params.put("floor", floor)
}
if(height){
params.put("maxFloor", height)
}
String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8");
JSONObject result = JSONObject.fromObject(response);
if (result.getInt("code") == 200) {