Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_v2
This commit is contained in:
+6
-8
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.open.interceptors;
|
||||
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.core.util.JsonUtils;
|
||||
import com.fangjia.dal.mongo.open.model.OpenResource;
|
||||
import com.fangjia.dal.mongo.tool.model.OpenUriLog;
|
||||
import com.fangjia.dal.sql.model.ApiUser;
|
||||
@@ -12,6 +13,8 @@ import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.service.open.OpenResourceService;
|
||||
import com.fangjia.service.open.OpenUriLogService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import net.sf.json.JSONNull;
|
||||
import net.sf.json.JSONObject;
|
||||
import net.sf.json.JsonConfig;
|
||||
import net.sf.json.processors.JsonValueProcessor;
|
||||
@@ -22,17 +25,13 @@ import org.sothis.web.mvc.ActionContext;
|
||||
import org.sothis.web.mvc.ActionInvocation;
|
||||
import org.sothis.web.mvc.ActionInvocationException;
|
||||
import org.sothis.web.mvc.Interceptor;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.math.BigDecimal;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
||||
|
||||
/**
|
||||
* API审计日志监控器
|
||||
*
|
||||
@@ -116,7 +115,6 @@ public class AuditInterceptor implements Interceptor {
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param request
|
||||
* @param apiUser
|
||||
@@ -163,7 +161,7 @@ public class AuditInterceptor implements Interceptor {
|
||||
if(isSaveResult(request)){
|
||||
JSONObject json = JSONObject.fromObject(result, DEFAULT_JSON_CONFIG);
|
||||
json.remove("tid");
|
||||
openUriLog.setResMap(json);
|
||||
openUriLog.setResMap((Map<String, Object>)JsonUtils.convertJsonToJavaObj(json, Map.class));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("", e);
|
||||
|
||||
+93
-1
@@ -813,8 +813,36 @@ class CommonDatautils {
|
||||
// q.put("status",new BasicDBObject('$ne', 0))
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
|
||||
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||
// q.put("name","经纬天地园二期")
|
||||
// q.put("name","扬州北街8号")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findNewUpdateDistricts(def city, def cat, def fields){
|
||||
|
||||
def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
// q.put("status",new BasicDBObject('$ne', 0))
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||
// q.put("name","扬州北街8号")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -5817,4 +5845,68 @@ class CommonDatautils {
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
static def findSeawater(city, lng, lat, distance, p_date, cat){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"geo_distance": {
|
||||
"distance": "${distance}",
|
||||
"location": {
|
||||
"lat": ${lat},
|
||||
"lon": ${lng}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"city": "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"cat": "${cat}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"range": {
|
||||
"publish_date": {
|
||||
"gte": "${p_date}",
|
||||
"format": "yyyy-MM-dd"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"sort": [
|
||||
{
|
||||
"publish_date": {
|
||||
"order": "desc"
|
||||
}
|
||||
}
|
||||
],
|
||||
"size":30
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
|
||||
def esObj = null
|
||||
if(response && response.hits){
|
||||
esObj = response.hits.hits
|
||||
}
|
||||
return esObj
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+1
@@ -211,6 +211,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def getFzDistricts(def city, def mdate, def endMDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
if(mdate){
|
||||
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
|
||||
}
|
||||
|
||||
+1
@@ -99,6 +99,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def getFzDistricts(def city, def mdate, def endMDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
if(mdate){
|
||||
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
|
||||
}
|
||||
|
||||
+39
-19
@@ -37,14 +37,14 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// def months = DateUtils.getMonthList(202206)
|
||||
// months = months[0..(months.size()-2)]
|
||||
// println months
|
||||
// for(def month : months){
|
||||
// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
|
||||
// calcMonthlyPrice(curDate)
|
||||
// }
|
||||
calcMonthlyPrice()
|
||||
def months = DateUtils.getMonthList(202510)
|
||||
months = months[0..(months.size()-1)]
|
||||
println months
|
||||
for(def month : months){
|
||||
def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
|
||||
calcMonthlyPrice(curDate)
|
||||
}
|
||||
// calcMonthlyPrice()
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,9 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
if(!curDate){
|
||||
curDate = new Date()
|
||||
}
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["上海"]//CommonUtils.findAllCity()
|
||||
// citys.remove("上海")
|
||||
// citys.remove("福州")
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
||||
|
||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate))
|
||||
@@ -87,22 +89,23 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
for(def city : citys){
|
||||
|
||||
index++
|
||||
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
if(day == 2){
|
||||
if(day == 29){
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
}else{
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
|
||||
}
|
||||
}else{
|
||||
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
|
||||
// q.put("name", "翠园楼")
|
||||
if(day == 2){
|
||||
// q.put("name", "越华路小区")
|
||||
// if(day == 30){
|
||||
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList()
|
||||
}else{
|
||||
q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
|
||||
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
|
||||
}
|
||||
// }else{
|
||||
// q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
|
||||
// districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
|
||||
// }
|
||||
}
|
||||
|
||||
println index+" "+city +" "+districts.size()+" "+ curDate.toString()
|
||||
@@ -155,7 +158,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def key = dist.region +"##" + dist.name
|
||||
|
||||
def weekendPriceObj = weekendPriceMap.get(key)
|
||||
if(weekendPriceObj){
|
||||
if(weekendPriceObj && weekendPriceObj.avg_price){
|
||||
def curPriceObj = curMonthlyPriceMap.get(key)
|
||||
def lastMonthPriceObj = lastMonthlyPriceMap.get(key)
|
||||
def lastYearPriceObj = lastYearPriceMap.get(key)
|
||||
@@ -216,8 +219,19 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", curMonth)
|
||||
q.put("commid", dist._id.toString())
|
||||
if(curPriceObj.commid){
|
||||
q.put("date", curMonth)
|
||||
q.put("commid", dist._id.toString())
|
||||
}else{
|
||||
q.put("date", curMonth)
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("district_name", dist.name)
|
||||
|
||||
data.put("commid", dist._id.toString())
|
||||
}
|
||||
|
||||
|
||||
DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false)
|
||||
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||
println curMonth+" [update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
@@ -239,6 +253,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
DISTRICT_SELL_STAT.save(data)
|
||||
println curMonth+" [save]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
@@ -281,6 +297,10 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
if(d.avg_price_2b && d.value_type!=1){
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}else{
|
||||
if(d.avg_price_2b){
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
}
|
||||
|
||||
@@ -831,7 +831,7 @@ class OpenApiUtils {
|
||||
static def addrVerifyGuangda(city, address){
|
||||
address = URLEncoder.encode(address, "utf-8");
|
||||
def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
|
||||
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
// def url = "http://localhost:8080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
|
||||
if(city){
|
||||
url = url + "&city=${city}"
|
||||
}
|
||||
@@ -847,23 +847,6 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
static def addrVerifyShBank(city, address){
|
||||
address = URLEncoder.encode(address, "utf-8");
|
||||
// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
|
||||
def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
if(city){
|
||||
url = url + "&city=${city}"
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
static def addrVerifyBase(city, address){
|
||||
@@ -945,4 +928,17 @@ class OpenApiUtils {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
static def lactionEvaluation(area, lng, lat){
|
||||
def url = "http://192.168.0.100:30020/avmWeb/avmAnn?area=${area}&location=lng"+","+lat
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.msgCode ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+58
-20
@@ -36,7 +36,8 @@ class SuzouPocTest {
|
||||
// task.newRun()
|
||||
// task.test()//有楼层
|
||||
// task.testEval()//无楼层
|
||||
task.level()//楼盘评级
|
||||
// task.level()//楼盘评级
|
||||
task.house()
|
||||
}
|
||||
|
||||
def test(){
|
||||
@@ -97,36 +98,73 @@ class SuzouPocTest {
|
||||
}
|
||||
|
||||
def testEval(){
|
||||
ExcelUtils.modify("D:\\数据\\其它\\附件1:POC测试样本(匹配).xlsx", { d ->
|
||||
if(d['O'] && !d['T']){
|
||||
ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d ->
|
||||
if(!d['O'] && d['E']){
|
||||
// def city = d['C']
|
||||
// if(city == "市辖区"){
|
||||
// city = d['B']
|
||||
// }
|
||||
def obj = CommonDatautils.findDistrictById(d['O'])
|
||||
if(obj){
|
||||
d['R'] = obj.city
|
||||
d['S'] = obj.region
|
||||
d['T'] = obj.name
|
||||
def ret1 = OpenApiClient.addressEvaluation(obj.city, obj.region, obj.city+obj.region+obj.name, obj.name, d['F'], null, null)
|
||||
// def obj = CommonDatautils.findDistrictById(d['O'])
|
||||
// if(obj){
|
||||
// d['R'] = obj.city
|
||||
// d['S'] = obj.region
|
||||
// d['T'] = obj.name
|
||||
def ret1 = OpenApiClient.addressEvaluation(d['C'], d['D'], d['C']+d['D']+d['E'], d['E'], d['I'], null, null)
|
||||
if (ret1) {
|
||||
d['P'] = ret1.totalPrice
|
||||
d['Q'] = ret1.avgPrice
|
||||
}else{
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name)
|
||||
if(dbo) {
|
||||
d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble())
|
||||
d['Q'] = dbo.avg_price_2b
|
||||
|
||||
}
|
||||
|
||||
d['J'] = ret1.totalPrice
|
||||
d['K'] = ret1.avgPrice
|
||||
d['L'] = ret1.city
|
||||
d['M'] = ret1.district
|
||||
d['N'] = ret1.name
|
||||
}
|
||||
}
|
||||
// else{
|
||||
// def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name)
|
||||
// if(dbo) {
|
||||
// d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble())
|
||||
// d['Q'] = dbo.avg_price_2b
|
||||
//
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def house(){
|
||||
def cases = []
|
||||
cases << ["城市", "区域","板块", "名称", "面积", "单价","总价","装修", "朝向", "距离","所在楼层","总楼层","室","厅","卫","发布时间","标题", "序号"]
|
||||
ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d ->
|
||||
if(d['L']){
|
||||
def obj = CommonDatautils.findDistrict(d['L'], d['M'], d['N'], "district")
|
||||
if(obj){
|
||||
def lng = obj.lng/10000000000.0
|
||||
def lat = obj.lat/10000000000.0
|
||||
def location = String.valueOf(lng)+","+ String.valueOf(lat)
|
||||
def result = OpenApiClient.houseAroundBylocationAction(location, d['I'], 0, 0, 50)
|
||||
if(result){
|
||||
for(def h : result){
|
||||
cases << [h.city, h.district, h.block, h.name, h.size, h.unitPrice, h.totalPrice, h.decoration, h.direction, h.distance, h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.publishDate, h.title, d["A"],"挂牌"]
|
||||
}
|
||||
}
|
||||
def address = d['L']+d['M']+d['N']
|
||||
def dealRets = OpenApiClient.dealHouse(d['L'], d['M'], d['N'], address, d['I'])
|
||||
if(dealRets){
|
||||
for(def h : dealRets){
|
||||
cases << [h.city, h.district, "", h.name, h.size, h.avgPrice, h.totalPrice, h.decoration, h.direction, "", h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.tradeDate, "", d["A"], "成交"]
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
println cases.size()
|
||||
ExcelUtils.write("D:\\数据\\宏信息\\样本房源-估值.xlsx", cases, "样本")
|
||||
}
|
||||
|
||||
def newRun(){
|
||||
|
||||
println("start:"+new Date().toString())
|
||||
|
||||
-2
@@ -38,8 +38,6 @@ class DevUpdateTestBatch {
|
||||
//
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDev(path){
|
||||
|
||||
+1
-1
@@ -191,7 +191,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
if(smallArr && smallArr.size()>=1){
|
||||
saveObj.small_type = smallArr[0]
|
||||
}else{
|
||||
saveObj.small_type = smallstr
|
||||
saveObj.8 = smallstr
|
||||
}
|
||||
}else{
|
||||
saveObj.small_type = saveObj.secondary_type
|
||||
|
||||
+2
-2
@@ -57,9 +57,9 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def TownStandard(){
|
||||
townStandardByCat("office")
|
||||
// townStandardByCat("office")
|
||||
// townStandardByPoi()
|
||||
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
// standardTown()
|
||||
// updateTownData()
|
||||
|
||||
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import net.sf.json.JSONNull
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
|
||||
class DistrictLocationGeo {
|
||||
|
||||
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
|
||||
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationGeo()
|
||||
task.run()//通过经纬度,调用高德逆解析,并保存数据
|
||||
// task.run2();//通过地址,调用高德逆解析
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
|
||||
if(d['E']){
|
||||
def jsonObject = GaoDeApiUtils.geocode(d['E'])
|
||||
if(jsonObject){
|
||||
d["F"] = "高德"
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.put("old_id", d['A'])
|
||||
dbo.put("city", d['B'])
|
||||
dbo.put("region", d['C'])
|
||||
dbo.put("name", d['D'])
|
||||
dbo.put("old_location", d['E'])
|
||||
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
|
||||
def addrComp = jsonObject.regeocode.addressComponent
|
||||
// addrComp.remove("neighborhood")
|
||||
// addrComp.remove("building")
|
||||
dbo.put("addressComponent", addrComp)
|
||||
dbo.put("pois", jsonObject.regeocode.pois)
|
||||
dbo.put("roads", jsonObject.regeocode.roads)
|
||||
dbo.put("roadinters", jsonObject.regeocode.roadinters)
|
||||
dbo.put("aois", jsonObject.regeocode.aois)
|
||||
dbo = cleanGeoData(dbo)
|
||||
dbo = convertToDBObject(dbo)
|
||||
println(dbo)
|
||||
|
||||
allSeaweedGaodePoisColl.save(dbo)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
static BasicDBObject convertToDBObject(Map map) {
|
||||
if (!map) return null
|
||||
|
||||
BasicDBObject dbObject = new BasicDBObject()
|
||||
map.each { key, value ->
|
||||
def convertedValue = convertValue(value)
|
||||
if (convertedValue != null) {
|
||||
dbObject.put(key, convertedValue)
|
||||
}
|
||||
}
|
||||
return dbObject
|
||||
}
|
||||
|
||||
static Object convertValue(Object value) {
|
||||
if (value == null) return null
|
||||
|
||||
switch (value) {
|
||||
case Map:
|
||||
return convertToDBObject(value as Map)
|
||||
case List:
|
||||
return convertList(value as List)
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
static BasicDBList convertList(List list) {
|
||||
BasicDBList dbList = new BasicDBList()
|
||||
list.each { item ->
|
||||
def convertedItem = convertValue(item)
|
||||
if (convertedItem != null) {
|
||||
dbList.add(convertedItem)
|
||||
}
|
||||
}
|
||||
return dbList
|
||||
}
|
||||
|
||||
static def cleanGeoData(data) {
|
||||
if (data == null) return null
|
||||
|
||||
def result = [:]
|
||||
|
||||
data.each { key, value ->
|
||||
def cleanedValue = cleanValue(value)
|
||||
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
|
||||
result[key] = cleanedValue
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static def cleanValue(value) {
|
||||
if (value == null) return null
|
||||
|
||||
switch (value) {
|
||||
case Map:
|
||||
def cleanedMap = [:]
|
||||
value.each { k, v ->
|
||||
def cleaned = cleanValue(v)
|
||||
if (cleaned != null && !isEmptyCollection(cleaned)) {
|
||||
cleanedMap[k] = cleaned
|
||||
}
|
||||
}
|
||||
return cleanedMap ? cleanedMap : null
|
||||
|
||||
case List:
|
||||
def cleanedList = value.collect { cleanValue(it) }
|
||||
.findAll { it != null && !isEmptyCollection(it) }
|
||||
return cleanedList ? cleanedList : null
|
||||
|
||||
case JSONNull:
|
||||
return null
|
||||
|
||||
default:
|
||||
// 检查是否是包含JSONNull的字符串表示
|
||||
if (value instanceof String && value.contains("JSONNull")) {
|
||||
return null
|
||||
}
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
static def isEmptyCollection(obj) {
|
||||
if (obj instanceof Map) return obj.isEmpty()
|
||||
if (obj instanceof List) return obj.isEmpty()
|
||||
return false
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d->
|
||||
def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B'])
|
||||
Thread.sleep(1000)
|
||||
def count = 0
|
||||
if (jsonObject) {
|
||||
def rets = jsonObject.geocodes
|
||||
if(rets){
|
||||
def dbo = rets[0]
|
||||
if(dbo){
|
||||
d['D'] = dbo.formatted_address
|
||||
d['E'] = dbo.level
|
||||
}
|
||||
}
|
||||
count = jsonObject.regeocode.pois.size()
|
||||
}
|
||||
d['D'] = count
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -355,7 +355,7 @@ class FangjiaClientUtils {
|
||||
address = address.toString().replace("{","")
|
||||
int retryCount = 0
|
||||
while (retryCount < 3) {
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
|
||||
url = "http://192.168.0.100:32080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+5
-10
@@ -6,17 +6,15 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.google.common.hash.*
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
@@ -24,15 +22,12 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
@@ -201,7 +196,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
@@ -492,8 +487,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
// def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
|
||||
+21
-4
@@ -1,19 +1,36 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class KunlunVillage {
|
||||
static main(args) {
|
||||
def task = new KunlunVillage();
|
||||
task.run()
|
||||
// task.addrCheck()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\昆仑\\3987户需批量更新涉农信息明细-20251023-结果.xlsx",{ d->
|
||||
def addrCheck(){
|
||||
|
||||
if(!d['R'] && d['B']){
|
||||
def dto = OpenApiUtils.addrVerifyGuangda(null, d['B'])
|
||||
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
|
||||
def obj = FangjiaClientUtils.formatNinelevel(null, d["C"])
|
||||
if(obj){
|
||||
println obj
|
||||
d['AB'] = obj.addressAssessment.msg
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
|
||||
index++
|
||||
if(!d['R'] && d['C']){
|
||||
println index
|
||||
def dto = OpenApiUtils.addrVerifyBase(null, d['C'])
|
||||
if(dto){
|
||||
// def dto = jsonObj.result
|
||||
d['F'] = dto.province
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ class XiamenVillage {
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-09-30)-结果.xlsx",{ d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-10-31)-以此为准.xlsx",{ d->
|
||||
|
||||
if(!d['R']){
|
||||
def dto = OpenApiUtils.addrVerifyXiameng(null, d['B'], 2)
|
||||
|
||||
+12
-10
@@ -1,21 +1,16 @@
|
||||
package com.fangjia.script.house
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{
|
||||
for(def city : citys){
|
||||
index ++ ;
|
||||
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
|
||||
def sellRet = houseStatCount(districts, startDate, endDate, sDate);
|
||||
|
||||
String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd")
|
||||
|
||||
def sellRet = houseStatCount(districts, start.toInteger(), sDate);
|
||||
saveResult(city, sDate, sellRet);
|
||||
|
||||
println "[${city} ${index}/${citys.size()}]" +new Date().toString();
|
||||
}
|
||||
}
|
||||
|
||||
def houseStatCount(def districts, def startDate, def endDate, def sDate){
|
||||
def houseStatCount(def districts, def startDate, def sDate){
|
||||
|
||||
def index = 0;
|
||||
|
||||
@@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{
|
||||
void run() {
|
||||
index++;
|
||||
def map = [:];
|
||||
map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
|
||||
map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
|
||||
// map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
|
||||
// map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
|
||||
|
||||
map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate)
|
||||
map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate)
|
||||
|
||||
map.city = dist.city;
|
||||
map.region = dist.region;
|
||||
map.name = dist.name;
|
||||
|
||||
+76
-27
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.jiangnanblank
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* Created by gaojuan on 2017/12/11.
|
||||
* 批量全国小区及商铺字段转换
|
||||
@@ -33,7 +27,7 @@ import java.util.regex.Pattern
|
||||
class CZDataBatch extends BaseBatch{
|
||||
|
||||
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
||||
def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
||||
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
|
||||
|
||||
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
|
||||
@@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println "args ²ÎÊý£º"+args.toString()
|
||||
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
citys.remove("常州");
|
||||
@@ -56,13 +52,22 @@ class CZDataBatch extends BaseBatch{
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
|
||||
def commids = findAllJnBankDatas(city)
|
||||
|
||||
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
||||
FzDataUtils.removeNoUpdate(city, districts)
|
||||
//
|
||||
def districts = null;
|
||||
def shops = null;
|
||||
if(args && args[0]== "all"){
|
||||
districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
||||
FzDataUtils.removeNoUpdate(city, districts)
|
||||
}else{
|
||||
districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields));
|
||||
}
|
||||
districtRun(districts, statMap, commids);
|
||||
|
||||
|
||||
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
||||
if(args && args[0]== "all"){
|
||||
shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
||||
}else{
|
||||
shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields);
|
||||
}
|
||||
shops = mergeShop(shops);
|
||||
shopRun(shops);
|
||||
}
|
||||
@@ -219,19 +224,22 @@ class CZDataBatch extends BaseBatch{
|
||||
// 商铺名称
|
||||
map.name = shop.name
|
||||
map.alias = shop.alias
|
||||
map.build_date = getBuildDate(shop);
|
||||
map.build_date = FzDataUtils.getBuildDate(shop);
|
||||
//附属属性
|
||||
map.build_type = shop.build_type
|
||||
map.property_life = shop.property_life
|
||||
map.estate_type2 = shop.estate_type2
|
||||
map.build_structure = shop.build_structure
|
||||
// 楼盘均价
|
||||
map.avg_price = getShopPrice(priceString, shop.city, shop.region, shop.name);
|
||||
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
|
||||
if(!map.avg_price){
|
||||
map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
|
||||
}
|
||||
// 省 市 区
|
||||
if(shop.city == "常州"){
|
||||
map.province_code = regionMap.get("˝ËŐ");
|
||||
map.city_code = regionMap.get(shop.city);
|
||||
map.region_code = regionMap.get(shop.region);
|
||||
map.province_code = FzDataUtils.regionMap.get("½ËÕ");
|
||||
map.city_code = FzDataUtils.regionMap.get(shop.city);
|
||||
map.region_code = FzDataUtils.regionMap.get(shop.region);
|
||||
map.province = "江苏";
|
||||
map.price_old_string = priceString;
|
||||
map.heights = heights;
|
||||
@@ -241,32 +249,73 @@ class CZDataBatch extends BaseBatch{
|
||||
// 地址
|
||||
map.address = shop.address
|
||||
// 所属区域
|
||||
map.in_region_code = getRegionCode(shop.region);
|
||||
map.in_region_code = FzDataUtils.getRegionCode(shop.region);
|
||||
// 商铺类型
|
||||
map.estate_type = getShopEstateTypeCode(shop.estate_type2);
|
||||
map.estate_type = FzDataUtils.getShopEstateTypeCode(shop.estate_type2);
|
||||
// 公共服务配套
|
||||
map.public_service_set_code = getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2);
|
||||
map.public_service_set_code = FzDataUtils.getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2);
|
||||
// 基础设施
|
||||
map.facilities_code = "01";
|
||||
// 商业密度
|
||||
map.sophistication_code = getSophisticationCode(shop.city, shop.lat2, shop.lng2);
|
||||
map.sophistication_code = FzDataUtils.getSophisticationCode(shop.city, shop.lat2, shop.lng2);
|
||||
|
||||
// 交通条件
|
||||
map.traffic_code = getTrafficCode(shop.city, shop.lat2, shop.lng2);
|
||||
map.traffic_code = FzDataUtils.getTrafficCode(shop.city, shop.lat2, shop.lng2);
|
||||
// 泊车条件 /
|
||||
map.stop_car_code = "01";
|
||||
// 临街道路
|
||||
map.street_code = getStreetCode(street);
|
||||
map.street_code = FzDataUtils.getStreetCode(street);
|
||||
// 租金
|
||||
map.shop_rent = getRentNumber(rentPrice, shop.city, shop.region, shop.name);
|
||||
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
|
||||
if(!map.shop_rent){
|
||||
map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
|
||||
}
|
||||
// 物业公司
|
||||
map.estate_company = getEstateCompany(shop.estate_company);
|
||||
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
|
||||
// 物业费(每月每平米)
|
||||
map.estate_charge = getEstateCharge(shop.estate_charge);
|
||||
map.estate_charge = FzDataUtils.getEstateCharge(shop.estate_charge);
|
||||
|
||||
saveShop(map);
|
||||
}
|
||||
|
||||
def getSellPrice(def city, def lng, def lat){
|
||||
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
|
||||
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell")
|
||||
if(rets){
|
||||
def prices = []
|
||||
for(def r : rets){
|
||||
prices << r._source.avg_price
|
||||
}
|
||||
if(prices){
|
||||
def price = (int)(prices.sum()/prices.size())
|
||||
return price
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getRentPrice(def city, def lng, def lat){
|
||||
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
|
||||
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent")
|
||||
if(!rets){
|
||||
rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent")
|
||||
if(!rets){
|
||||
rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent")
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
def prices = []
|
||||
for(def r : rets){
|
||||
prices << r._source.avg_price
|
||||
}
|
||||
if(prices){
|
||||
def day_price = (prices.sum()/prices.size())/30
|
||||
return DataHandleUtils.getDoubleRound(2, day_price)
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveShop(def m){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city",m.city)
|
||||
|
||||
+1
@@ -809,4 +809,5 @@ class FzDataUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+19
-2
@@ -20,10 +20,12 @@ class TestEvaluation {
|
||||
|
||||
// task.match()
|
||||
// task.houseSimilar()
|
||||
task.newMatch()
|
||||
// task.newMatch()
|
||||
// task.houseSimilar()
|
||||
|
||||
task.newAndOldMatch()
|
||||
// task.newAndOldMatch()
|
||||
|
||||
task.addFjwToBprice("D:\\数据\\估值\\地址估价-匹配(加写字楼-商铺) - v2.xlsx")
|
||||
}
|
||||
|
||||
def houseSimilar(){
|
||||
@@ -206,4 +208,19 @@ class TestEvaluation {
|
||||
})
|
||||
}
|
||||
|
||||
def addFjwToBprice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
if(dbo.avg_price_2b){
|
||||
d['H'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 6
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 8
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.SystemUtils
|
||||
|
||||
@@ -1577,5 +1578,73 @@ public class OpenApiClient {
|
||||
|
||||
|
||||
|
||||
public static def houseAroundBylocationAction(location, size, propertyType, houseType, pageSize) {
|
||||
|
||||
def url ="https://open.fangjia.com/house/around?location="+location+"&size="+size+"&pageSize="+pageSize+"&propertyType="+propertyType+"&houseType="+houseType
|
||||
url = url +"&token="+TOKEN
|
||||
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def dealHouse(city, region, name, area, address){
|
||||
|
||||
def url ="https://open.fangjia.com/property/deal?city="+city+"&address="+address+"&district="+region+"&name="+name+"&size="+area
|
||||
url = url +"&token="+TOKEN
|
||||
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject propertyHouseAction(city, district, name, size, propertyType) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("district", district)
|
||||
params.put("name", name)
|
||||
params.put("size", size.toString());
|
||||
params.put("token", TOKEN);
|
||||
params.put("propertyType", propertyType)
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("https://open.fangjia.com/property/house", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/property/house", params, "utf-8")
|
||||
println("${city} ${name} ³ÌÐòµ÷ÓÃÒì³£")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if (result.getInt("code") == 0) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${city} ${name} ")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${name} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+42
-4
@@ -1,19 +1,17 @@
|
||||
package com.fangjia.script.yinjihaun.amap
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
//94c74c60e102972a8d17b7b75d7403be
|
||||
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390
|
||||
//67423021cae3b58e5c2ac595a389b036 :临时买的key
|
||||
def static GAO_DE_AK = "67423021cae3b58e5c2ac595a389b036" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
|
||||
//15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
@@ -29,6 +27,11 @@ class GaoDeApiUtils {
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("location", location)
|
||||
params.put("extensions","all")
|
||||
params.put("radius", "3000")
|
||||
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")//纯住宅类型
|
||||
|
||||
//京东测试允许类型
|
||||
params.put("poitype", "100000|100100|100101|100102|100103|100104|100105|100200|100201|120000|120200|120203|120300|120301|120302|120303|120304|190000|190100|190400|190401|190402|190403|190500|190600|190700")
|
||||
String content
|
||||
def stop = true
|
||||
JSONObject jsonObj = null
|
||||
@@ -37,6 +40,7 @@ class GaoDeApiUtils {
|
||||
try {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
println location + " 出错:"+content
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
}
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
@@ -55,6 +59,40 @@ class GaoDeApiUtils {
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
def static geocodeByAddress(String city, String address) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("address", address)
|
||||
params.put("batch", "true")
|
||||
params.put("city", city)
|
||||
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")
|
||||
String content
|
||||
def stop = true
|
||||
JSONObject jsonObj = null
|
||||
def errorCount = 1
|
||||
while (stop){
|
||||
try {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
|
||||
}
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
if(jsonObj.status!="1"){
|
||||
println jsonObj
|
||||
errorCount ++
|
||||
Thread.sleep(3000)
|
||||
if(errorCount>3){
|
||||
stop = false
|
||||
}
|
||||
}else{
|
||||
stop = false
|
||||
}
|
||||
}
|
||||
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
def static geo(city, address) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü
|
||||
|
||||
+6
-2
@@ -339,9 +339,13 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
}
|
||||
}
|
||||
district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS);
|
||||
if(district == null){
|
||||
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + district.getId());
|
||||
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId());
|
||||
}else{
|
||||
district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS);
|
||||
}
|
||||
if(district == null){
|
||||
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId());
|
||||
}
|
||||
return new Building(district, scoreResult.getScore(matchedMate));
|
||||
}else{
|
||||
|
||||
Reference in New Issue
Block a user