脚本修改

This commit is contained in:
2023-10-30 16:30:53 +08:00
parent 42af88d063
commit a5e2033e1b
12 changed files with 557 additions and 75 deletions
@@ -679,4 +679,24 @@ class CcbCommonUtils {
return ids;
}
static def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
}
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202308
def month = 202309
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202309)
def rets = findPeGteThirdParty2(city, 202310)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1288,30 +1288,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
//// def region = RegionData.formatRegion(d['B'], d['C'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//// if(dist){
//// d['G'] = dist.region
//// d['H'] = dist._id.toString()
//// }
//// }
// if(!d['K']){
// def dist = CommonDatautils.findDistrictById(d['E'])
// if(dist){
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// d['K'] = dist.region
//// d['I'] = dist.name
//// d['J'] = dist.status
//// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
//// d['K'] = val.toInteger()
// }
// }
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
d['K'] = dist.region
// d['I'] = dist.name
// d['J'] = dist.status
// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
// d['K'] = val.toInteger()
}
}
}
},0)
// }
//},0)
@@ -1415,6 +1415,14 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\23号以后新增人工指导价.xlsx", {d->
def obj = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
d['H'] = obj._id.toString()
})
@@ -19,10 +19,12 @@ class PocTest {
static main(args) {
def task = new PocTest();
task.run();
// task.run();
// task.export()
// task.distance()
}
def export(){
ExcelUtils.modify("D:\\数据\\其它\\广州.xlsx", { row ->
def dist = CommonDatautils.findDistrictById(row["B"])
@@ -156,7 +158,6 @@ class PocTest {
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['R'] = priceObj.avg_price
@@ -87,6 +87,7 @@ class CommonDatautils {
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
@@ -250,10 +251,10 @@ class CommonDatautils {
List<DBObject> dbs = cursor.getResults();
return dbs;
}
public static def findAllDistricts(def city, def cat, def fields){
public static def findAllDistrictsFromMg(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","文华楼")
@@ -265,7 +266,7 @@ class CommonDatautils {
}
}
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -793,6 +794,29 @@ class CommonDatautils {
}
public static def findAllDistricts(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city)
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistricMaptByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
@@ -2628,6 +2652,7 @@ class CommonDatautils {
q.put("cat", cat);
q.put('lat2', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
q.put('lng2', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
q.put("visible", 0)
def cur = getDataStoreConnection().find(q);
def ret = [];
for(def obj : cur){
@@ -3886,42 +3911,6 @@ class CommonDatautils {
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
/**
@@ -4149,4 +4138,86 @@ class CommonDatautils {
}
/**
* 获取小区周边一公里小区数量
* @param city
* @param lng
* @param lat
* @return
*/
static getDistrictArround(city, lng, lat) {
try {
def query = new BasicDBObject()
query.append("cat", "district")
query.append("city", city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
query.append('$sd', sdq)
return DbUtils.seaweed.find(query).toList()
} catch (Exception e) {
e.printStackTrace()
}
return 0
}
static def findThirdPartyPrice(def city, def region, def name, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
static def findGXDPrice(def city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
for(def d : dbcursor){
price = d.price
}
return price
}
}
@@ -3,6 +3,7 @@ package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
@@ -25,7 +26,18 @@ class AddAvgpriceBatch {
static main(args) {
def task = new AddAvgpriceBatch();
// task.run1();
task.test()
// task.test()
task.sendPriceCommand()
}
def sendPriceCommand(){
ExcelUtils.multiThreadingModify("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx", { row ->
if(!row['P'] && row['T'] && ["别墅","普通住宅","商住两用"].contains(row['T'])){
//发重算均价命令
println row['K']+" "+row['L']+" "+row['M']
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
}
})
}
def test(){
@@ -207,7 +207,14 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&category=${property}&useMap=${useMap}"
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
if(property){
url = url + "&category=${property}"
}
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
@@ -218,4 +225,19 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def newStandard(address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -0,0 +1,227 @@
package com.fangjia.script.gj.meishubao
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class MsbPocTest {
static main(args) {
def task = new MsbPocTest();
task.run();
}
def run(){
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
// matchOld(inputfile)
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
// test("D:\\数据\\美术宝\\test2.xlsx")
cityPrice()
}
def cityPrice(){
def datas = ExcelUtils.read("D:\\数据\\临时\\城市房产价格.xlsx",0,1)
def map = [:]
for(def d : datas){
map.put(d['A'], d)
}
def ccbdatas = ExcelUtils.read("D:\\数据\\临时\\建行城市均价.xlsx",0,1)
def ccbmap = [:]
for(def d : ccbdatas){
ccbmap.put(d['A'], d['B'])
}
ExcelUtils.modify("D:\\数据\\临时\\房价网城市均价.xlsx", { row ->
if(!row['C']){
def d = map.get(row['A'])
if(!d){
d = map.get(row['A'].replace('州',''))
}
if(d){
row['C'] = d[1]
row['D'] = d[2]
}
def d1 = ccbmap.get(row['A'])
if(!d1){
d1 = ccbmap.get(row['A'].replace('州',''))
}
if(d1){
row['E'] = d1
}
}
})
}
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['S'] && row['D']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['H']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['S'] = jsonObject.get("id")
row['T'] = jsonObject.get("city")
row['U'] = jsonObject.get("district")
row['V'] = jsonObject.get("name")
row['W'] = jsonObject.get("propertyType")
row['X'] = jsonObject.get("credit")
}
}
})
}
def test(file){
ExcelUtils.multiThreadingModify(file, { row ->
def jsonObject = FangjiaClientUtils.newMatch(null, row['A'], null, true)
println row['A']
if(jsonObject != null && jsonObject.credit>0) {
row['B'] = jsonObject.get("id")
row['C'] = jsonObject.get("city")
row['D'] = jsonObject.get("district")
row['E'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['F'] = type
}
row['G'] = jsonObject.get("credit")
}
})
}
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
def city = row['C']
def region = row['D']
def address = row['G'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
if(jsonObject != null && jsonObject.credit>0) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['N'] = type
}
row['O'] = jsonObject.get("credit")
}
}
})
}
def addPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(row['I'] && !row['P']){
row['I'] = row['I'].toString().trim()
def d = CommonDatautils.findDistrictById(row['I'])
if(d){
row['K'] = d.city
row['L'] = d.region
row['M'] = d.name
row['T'] = d.estate_type2
row['U'] = d.cat
row['V'] = d.status
def cityMap = CommonDatautils.findThirdPartyPrice(d.city, d.region, d.name, 202303,202310)
def priceMap = cityMap.get(d.city+"##"+d.region+"##"+d.name)
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['N'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['N'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['O'] = priceMap.get("anjuke_esf_price")
}
}
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
if(dbo){
row['P'] = dbo.avg_price
}
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(row['I'], cityId)
if(map){
row['Q'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-10-01')
if(tradePrice){
row['R'] = tradePrice.toInteger()
}
def gxdPrice = CommonDatautils.findGXDPrice(d.city, d.region, d.name)
d['S'] = gxdPrice
}
}
})
}
def standard(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['G']){
row['G'] = row['F'].replace('·','').replace("小区","").replace("花园","").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
// println "[match] city:"+city+" region:"+region+" address:"+address
// def jsonObject = FangjiaClientUtils.newStandard(row['G'])
// if(jsonObject != null && jsonObject.precise!=-1) {
// row['Q'] = jsonObject.get("name")
// row['R'] = jsonObject.get("id")
// }
}
})
}
def addThirdPrice(path){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
})
}
}
@@ -0,0 +1,80 @@
package com.fangjia.script.gj.test
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
//def json = GaoDeApiUtils.geo("上海市", "上海市浦东新区永泰路1138弄11号")
//def locationStr = null
//if(json && json.count>=1){
// def m = json.geocodes[0]
// if(m){
// locationStr = m.location
// }
//}
def location = ["lng":121.526016,"lat":31.143047]
//def rows = []
//rows << ["城市","区域","名称","地址","距离","均价"]
//if(location && location.lat){
//
// def dbos = CommonDatautils.getDistrictArround("上海", 121.526016, 31.143047)
// for(def d : dbos){
//
// def dbo = CommonDatautils.findDistrictPrice(d.city, d.region, d.name);
// def avg_price = null
// if(dbo){
// avg_price = dbo.avg_price
// }
//
// def distance = GeoUtils.calculateDistance(location.lng, location.lat, d.lng/10000000000.0d, d.lat/10000000000.0)
//
// rows << [d.city, d.region, d.name, d.address, distance.intValue(), avg_price, d.status]
// }
//
//// def map = CommonDatautils.findMaxLatLngMinLatLng(location.lat, location.lng, 1000);
//// def ret = CommonDatautils.findInRangeDist("上海", "district", (map.maxLat * 10000000000d).toLong(), (map.minLat * 10000000000d).toLong(), (map.maxLng * 10000000000d).toLong(), (map.minLng * 10000000000d).toLong());
////
//// if(ret){
//// for(def row : ret){
//// if(row.lat2 && row.lng2){
//// def lat2 = row.lat2/10000000000.0;
//// def lng2 = row.lng2/10000000000.0;
//// region = row.region;
//// def distance = GeoHelper.GetDistance(location.lat, location.lng, lat2, lng2)
//// if(distance <= 1000){
//// def dbo = CommonDatautils.findDistrictPrice(row.city, row.region, row.name);
//// def avg_price = null
//// if(dbo){
//// avg_price = dbo.avg_price
//// }
//// rows << [row.city, row.region, row.name, row.address, distance.intValue(), avg_price]
//// }
//// }
//// }
//// }
//
// ExcelUtils.write("D:\\数据\\临时\\上海市浦东新区永泰路1138弄11号.xlsx", rows, "一公里楼盘")
//}
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{d->
def loc = d['AB']
if(loc){
def arrs = loc.split(",")
def distance = GeoUtils.calculateDistance(location.lng, location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
d['AD'] = distance
}
})
//def distance = GeoUtils.calculateDistance(121.526016, 31.143047, 111.202375, 35.592230)
//println distance
@@ -1,12 +1,14 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
@@ -16,13 +18,15 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch {
class SeaweedDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -30,14 +34,27 @@ class SeaweedDataSynch {
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
static main(args) {
def task = new SeaweedDataSynch()
// def addr = ["a","b",'c',"d","e","f"]
// println addr[0..4]
task.run()
// println new ObjectId("622c433afb459ab08fdab460").getDate()
runBatch(new SeaweedDataSynch(), args);
}
@Override
public void process(Object args) {
run()
println "全国district, shop, office同步完成"
//民政部编码上传
//codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
}
def run(){
priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity()
@@ -52,7 +69,7 @@ class SeaweedDataSynch {
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -208,4 +225,27 @@ class SeaweedDataSynch {
}
return priceMap
}
def codeRun(){
def q = new BasicDBObject()
def dbcursor = mca_administrative_division_coll.find(q)
def rets = []
for(def it : dbcursor){
def udata = new BasicDBObject()
udata.put("_id", it._id)
udata.put("province", it.province)
udata.put("city", it.city)
udata.put("district", it.district)
udata.put("mca_province_code", it.province_code)
udata.put("mca_city_code", it.city_code)
udata.put("mca_district_code", it.district_code)
udata.put("level", it.level)
udata.put("c_date", it.c_date)
rets << udata
// administrative_division_100_CONN.save(udata)
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
}
administrative_division_100_CONN.insert(rets)
}
}
@@ -414,7 +414,8 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 2
def nThread = Runtime.getRuntime().availableProcessors() * 5
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
final double total = body.size()