+20
@@ -679,4 +679,24 @@ class CcbCommonUtils {
|
||||
return ids;
|
||||
}
|
||||
|
||||
|
||||
static def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202308
|
||||
def month = 202309
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202309)
|
||||
def rets = findPeGteThirdParty2(city, 202310)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-27")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+28
-20
@@ -1288,30 +1288,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
//// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//// if(dist){
|
||||
//// d['G'] = dist.region
|
||||
//// d['H'] = dist._id.toString()
|
||||
//// }
|
||||
//// }
|
||||
// if(!d['K']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
// if(dist){
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// d['K'] = dist.region
|
||||
//// d['I'] = dist.name
|
||||
//// d['J'] = dist.status
|
||||
//// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
|
||||
//// d['K'] = val.toInteger()
|
||||
// }
|
||||
// }
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
d['K'] = dist.region
|
||||
// d['I'] = dist.name
|
||||
// d['J'] = dist.status
|
||||
// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
|
||||
// d['K'] = val.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
@@ -1415,6 +1415,14 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\23号以后新增人工指导价.xlsx", {d->
|
||||
def obj = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
|
||||
d['H'] = obj._id.toString()
|
||||
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -19,10 +19,12 @@ class PocTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new PocTest();
|
||||
task.run();
|
||||
// task.run();
|
||||
// task.export()
|
||||
// task.distance()
|
||||
}
|
||||
|
||||
|
||||
def export(){
|
||||
ExcelUtils.modify("D:\\数据\\其它\\广州.xlsx", { row ->
|
||||
def dist = CommonDatautils.findDistrictById(row["B"])
|
||||
@@ -156,7 +158,6 @@ class PocTest {
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
// row['Q'] = dist.name
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['R'] = priceObj.avg_price
|
||||
|
||||
+111
-40
@@ -87,6 +87,7 @@ class CommonDatautils {
|
||||
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
@@ -250,10 +251,10 @@ class CommonDatautils {
|
||||
List<DBObject> dbs = cursor.getResults();
|
||||
return dbs;
|
||||
}
|
||||
|
||||
public static def findAllDistricts(def city, def cat, def fields){
|
||||
|
||||
public static def findAllDistrictsFromMg(def city, def cat, def fields){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible", 0);
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","文华楼")
|
||||
@@ -265,7 +266,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -793,6 +794,29 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findAllDistricts(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistricMaptByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -2628,6 +2652,7 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put('lat2', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
|
||||
q.put('lng2', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
|
||||
q.put("visible", 0)
|
||||
def cur = getDataStoreConnection().find(q);
|
||||
def ret = [];
|
||||
for(def obj : cur){
|
||||
@@ -3886,42 +3911,6 @@ class CommonDatautils {
|
||||
return avg_price
|
||||
}
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
@@ -4149,4 +4138,86 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取小区周边一公里小区数量
|
||||
* @param city
|
||||
* @param lng
|
||||
* @param lat
|
||||
* @return
|
||||
*/
|
||||
static getDistrictArround(city, lng, lat) {
|
||||
try {
|
||||
def query = new BasicDBObject()
|
||||
query.append("cat", "district")
|
||||
query.append("city", city)
|
||||
query.append("visible", 0)
|
||||
DBObject sdq = new BasicDBObject();
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
|
||||
query.append('$sd', sdq)
|
||||
return DbUtils.seaweed.find(query).toList()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
static def findGXDPrice(def city, region, name){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("source", "GXD")
|
||||
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
|
||||
def price = null
|
||||
for(def d : dbcursor){
|
||||
price = d.price
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+13
-1
@@ -3,6 +3,7 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
@@ -25,7 +26,18 @@ class AddAvgpriceBatch {
|
||||
static main(args) {
|
||||
def task = new AddAvgpriceBatch();
|
||||
// task.run1();
|
||||
task.test()
|
||||
// task.test()
|
||||
task.sendPriceCommand()
|
||||
}
|
||||
|
||||
def sendPriceCommand(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx", { row ->
|
||||
if(!row['P'] && row['T'] && ["别墅","普通住宅","商住两用"].contains(row['T'])){
|
||||
//发重算均价命令
|
||||
println row['K']+" "+row['L']+" "+row['M']
|
||||
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def test(){
|
||||
|
||||
+23
-1
@@ -207,7 +207,14 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&category=${property}&useMap=${useMap}"
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
if(property){
|
||||
url = url + "&category=${property}"
|
||||
}
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
@@ -218,4 +225,19 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def newStandard(address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
package com.fangjia.script.gj.meishubao
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class MsbPocTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new MsbPocTest();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
|
||||
def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
// matchOld(inputfile)
|
||||
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
|
||||
// test("D:\\数据\\美术宝\\test2.xlsx")
|
||||
|
||||
cityPrice()
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\临时\\城市房产价格.xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['A'], d)
|
||||
}
|
||||
|
||||
def ccbdatas = ExcelUtils.read("D:\\数据\\临时\\建行城市均价.xlsx",0,1)
|
||||
def ccbmap = [:]
|
||||
for(def d : ccbdatas){
|
||||
ccbmap.put(d['A'], d['B'])
|
||||
}
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\临时\\房价网城市均价.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
def d = map.get(row['A'])
|
||||
if(!d){
|
||||
d = map.get(row['A'].replace('州',''))
|
||||
}
|
||||
if(d){
|
||||
row['C'] = d[1]
|
||||
row['D'] = d[2]
|
||||
}
|
||||
|
||||
|
||||
def d1 = ccbmap.get(row['A'])
|
||||
if(!d1){
|
||||
d1 = ccbmap.get(row['A'].replace('州',''))
|
||||
}
|
||||
if(d1){
|
||||
row['E'] = d1
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['S'] && row['D']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['H']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['S'] = jsonObject.get("id")
|
||||
row['T'] = jsonObject.get("city")
|
||||
row['U'] = jsonObject.get("district")
|
||||
row['V'] = jsonObject.get("name")
|
||||
row['W'] = jsonObject.get("propertyType")
|
||||
row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def test(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
def jsonObject = FangjiaClientUtils.newMatch(null, row['A'], null, true)
|
||||
println row['A']
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['B'] = jsonObject.get("id")
|
||||
row['C'] = jsonObject.get("city")
|
||||
row['D'] = jsonObject.get("district")
|
||||
row['E'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['F'] = type
|
||||
}
|
||||
row['G'] = jsonObject.get("credit")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['J'] = jsonObject.get("id")
|
||||
row['K'] = jsonObject.get("city")
|
||||
row['L'] = jsonObject.get("district")
|
||||
row['M'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['N'] = type
|
||||
}
|
||||
row['O'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['I'] && !row['P']){
|
||||
row['I'] = row['I'].toString().trim()
|
||||
def d = CommonDatautils.findDistrictById(row['I'])
|
||||
if(d){
|
||||
row['K'] = d.city
|
||||
row['L'] = d.region
|
||||
row['M'] = d.name
|
||||
row['T'] = d.estate_type2
|
||||
row['U'] = d.cat
|
||||
row['V'] = d.status
|
||||
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(d.city, d.region, d.name, 202303,202310)
|
||||
def priceMap = cityMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['N'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['N'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['O'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
|
||||
if(dbo){
|
||||
row['P'] = dbo.avg_price
|
||||
}
|
||||
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(row['I'], cityId)
|
||||
if(map){
|
||||
row['Q'] = map.avg_price
|
||||
}
|
||||
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-10-01')
|
||||
if(tradePrice){
|
||||
row['R'] = tradePrice.toInteger()
|
||||
}
|
||||
|
||||
def gxdPrice = CommonDatautils.findGXDPrice(d.city, d.region, d.name)
|
||||
d['S'] = gxdPrice
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def standard(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['G']){
|
||||
|
||||
row['G'] = row['F'].replace('·','').replace("小区","").replace("花园","").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
// println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
// def jsonObject = FangjiaClientUtils.newStandard(row['G'])
|
||||
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['Q'] = jsonObject.get("name")
|
||||
// row['R'] = jsonObject.get("id")
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(row['D'], row['E'], row['F'])
|
||||
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['L'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['M'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['N'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
|
||||
//def json = GaoDeApiUtils.geo("上海市", "上海市浦东新区永泰路1138弄11号")
|
||||
//def locationStr = null
|
||||
//if(json && json.count>=1){
|
||||
// def m = json.geocodes[0]
|
||||
// if(m){
|
||||
// locationStr = m.location
|
||||
// }
|
||||
//}
|
||||
|
||||
def location = ["lng":121.526016,"lat":31.143047]
|
||||
|
||||
|
||||
//def rows = []
|
||||
//rows << ["城市","区域","名称","地址","距离","均价"]
|
||||
//if(location && location.lat){
|
||||
//
|
||||
// def dbos = CommonDatautils.getDistrictArround("上海", 121.526016, 31.143047)
|
||||
// for(def d : dbos){
|
||||
//
|
||||
// def dbo = CommonDatautils.findDistrictPrice(d.city, d.region, d.name);
|
||||
// def avg_price = null
|
||||
// if(dbo){
|
||||
// avg_price = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def distance = GeoUtils.calculateDistance(location.lng, location.lat, d.lng/10000000000.0d, d.lat/10000000000.0)
|
||||
//
|
||||
// rows << [d.city, d.region, d.name, d.address, distance.intValue(), avg_price, d.status]
|
||||
// }
|
||||
//
|
||||
//// def map = CommonDatautils.findMaxLatLngMinLatLng(location.lat, location.lng, 1000);
|
||||
//// def ret = CommonDatautils.findInRangeDist("上海", "district", (map.maxLat * 10000000000d).toLong(), (map.minLat * 10000000000d).toLong(), (map.maxLng * 10000000000d).toLong(), (map.minLng * 10000000000d).toLong());
|
||||
////
|
||||
//// if(ret){
|
||||
//// for(def row : ret){
|
||||
//// if(row.lat2 && row.lng2){
|
||||
//// def lat2 = row.lat2/10000000000.0;
|
||||
//// def lng2 = row.lng2/10000000000.0;
|
||||
//// region = row.region;
|
||||
//// def distance = GeoHelper.GetDistance(location.lat, location.lng, lat2, lng2)
|
||||
//// if(distance <= 1000){
|
||||
//// def dbo = CommonDatautils.findDistrictPrice(row.city, row.region, row.name);
|
||||
//// def avg_price = null
|
||||
//// if(dbo){
|
||||
//// avg_price = dbo.avg_price
|
||||
//// }
|
||||
//// rows << [row.city, row.region, row.name, row.address, distance.intValue(), avg_price]
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// ExcelUtils.write("D:\\数据\\临时\\上海市浦东新区永泰路1138弄11号.xlsx", rows, "一公里楼盘")
|
||||
//}
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{d->
|
||||
def loc = d['AB']
|
||||
if(loc){
|
||||
def arrs = loc.split(",")
|
||||
def distance = GeoUtils.calculateDistance(location.lng, location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
|
||||
d['AD'] = distance
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
//def distance = GeoUtils.calculateDistance(121.526016, 31.143047, 111.202375, 35.592230)
|
||||
//println distance
|
||||
+47
-7
@@ -1,12 +1,14 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -16,13 +18,15 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
class SeaweedDataSynch {
|
||||
class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
@@ -30,14 +34,27 @@ class SeaweedDataSynch {
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new SeaweedDataSynch()
|
||||
// def addr = ["a","b",'c',"d","e","f"]
|
||||
// println addr[0..4]
|
||||
task.run()
|
||||
// println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
runBatch(new SeaweedDataSynch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run()
|
||||
println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
//codeRun()
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
@@ -52,7 +69,7 @@ class SeaweedDataSynch {
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
@@ -208,4 +225,27 @@ class SeaweedDataSynch {
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
def codeRun(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = mca_administrative_division_coll.find(q)
|
||||
def rets = []
|
||||
for(def it : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("_id", it._id)
|
||||
udata.put("province", it.province)
|
||||
udata.put("city", it.city)
|
||||
udata.put("district", it.district)
|
||||
udata.put("mca_province_code", it.province_code)
|
||||
udata.put("mca_city_code", it.city_code)
|
||||
udata.put("mca_district_code", it.district_code)
|
||||
udata.put("level", it.level)
|
||||
udata.put("c_date", it.c_date)
|
||||
rets << udata
|
||||
// administrative_division_100_CONN.save(udata)
|
||||
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
administrative_division_100_CONN.insert(rets)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,7 +414,8 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 2
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
final double total = body.size()
|
||||
|
||||
Reference in New Issue
Block a user