This commit is contained in:
2024-11-28 17:36:46 +08:00
parent f8bc248feb
commit d0cd6e575e
9 changed files with 428 additions and 130 deletions
@@ -1,15 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
@@ -54,8 +57,8 @@ class AddDistrictShapeBatch extends BaseBatch{
//
// }
// checkRangeCount()
synch100()
checkRangeCount()
// synch100()
}
def baiduMatch(city,districts){
@@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{
def checkRangeCount(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","乡镇街道")).batchSize(100)
def count = 0
def index =0
for(def d : dbcursor){
index ++
if(d.location){
def coordinates = d.location.coordinates
if(d.shape){
def coordinates = d.shape.coordinates
if(coordinates.size()>1){
count++
println count +" "+ d.city +" "+d.name
println count +" "+ d.city +" "+d.street
}
}else{
println d.city + " "+d.name
println d.city + " "+d.street
}
if(index%1000==0){
@@ -35,22 +35,22 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
// //查找第三方匹配没有匹配上的小区
// def citys = CommonDatautils.findAllCity()
// def list = []
// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
// for(def c : citys){
// def noMatchDatas = findNoMatchResult(c)
// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
// def rets = checkDatas(noMatchDatas.keySet())
// list.addAll(rets)
// }
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
fileName = "D:\\数据\\add_dist_info_1.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
// ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
@@ -61,6 +61,11 @@ class AutoAddDistrictBatch extends BaseBatch{
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中已存在"
}else{
dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中别名中存在"
}
}
})
}
@@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.joda.time.Instant
import java.time.format.DateTimeFormatter
import java.util.regex.Pattern
class DevProcess {
@@ -64,6 +58,7 @@ class DevProcess {
// task.deleteAddr()
// task.deleteAddr2()
task.synchDevTo100()
// task.statShape()
// task.addNewDev2()
// task.addAlias()
@@ -87,6 +82,8 @@ class DevProcess {
// task.matchDevShape()//匹配开发区围栏
// task.addQiDevResult() //获取齐梁数据结果
// task.updateDevShape()//修改开发区结构
}
@@ -794,6 +791,18 @@ class DevProcess {
development_zone_CONN.insert(rets)
}
def statShape(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true)))
def index = 0
for(def r : dbcursor){
index ++
def coordinates = r.shape.coordinates
println index+" "+r.name+" "+coordinates.size()
}
}
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
@@ -1148,4 +1157,26 @@ class DevProcess {
}
},0)
}
def updateDevShape(){
def q = new BasicDBObject()
q.put("shape", new BasicDBObject('$exists', true))
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q)
for(def d : dbcursor){
if(d.name!="福州长乐国际机场综合保税区"){
def udata = new BasicDBObject()
if(d.shape.coordinates){
d.shape.coordinates = [d.shape.coordinates]
udata.put("shape", d.shape)
}
if(d.bd_shape && d.bd_shape.coordinates){
d.bd_shape.coordinates = [d.bd_shape.coordinates]
udata.put("bd_shape", d.bd_shape)
}
def updateQuery = new BasicDBObject("_id",d._id)
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
@@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -33,7 +32,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
//
@@ -168,7 +167,7 @@ class DevUpdateTestBatch {
println d['A'] +" 不存在"
}
}
},2)
},0)
}
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class NewPoiData {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new NewPoiData()
task.run()
}
def run(){
// export()
// exportNetWorkOffice()
// matchByCity()
// noMatchData()
calcuteDistance()
}
def noMatchData(){
ExcelUtils.multiThreadingModify("d:\\数据\\未匹配写字楼2.xlsx", { d ->
if(d['H']!='地图'){
def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C'])
if(map){
d['E'] = map.lng
d['F'] = map.lat
}
}
})
}
def matchByCity(){
ExcelUtils.multiThreadingModify("d:\\数据\\入库写字楼.xlsx",{d->
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false)
def formatAddress = null
if(d['D']){
formatAddress = AddressUtil.formatAddress(d['A'], d['D'])
d['G'] = formatAddress
}
if(!jsonObject && formatAddress){
if(DataUtil.isPreciseAddress(formatAddress)){
def address = d['A']
if(d['B']){
address = address + d['B']
}
address = address + formatAddress
jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false)
}
}
if(jsonObject) {
d['H'] = jsonObject.get("id")
d['I'] = jsonObject.get("city")
d['J'] = jsonObject.get("district")
d['K'] = jsonObject.get("name")
d['L'] = jsonObject.get("propertyType")
d['M'] = jsonObject.get("credit")
}
})
}
def export(){
def rets = []
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
rets.addAll(CommonDatautils.findNewPoiDatas())
ExcelUtils.write("d:\\数据\\poi写字楼.xlsx", rets, "写字楼")
}
def exportNetWorkOffice(){
def rets = []
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
rets.addAll(CommonDatautils.findThirdNetWorkOffice())
ExcelUtils.write("d:\\数据\\poi写字楼-网站.xlsx", rets, "写字楼")
}
def calcuteDistance(){
ExcelUtils.multiThreadingModify("d:\\数据\\写字楼测试-1118-新增.xlsx", { d ->
if(d['F']){
if(d['A'].contains("北京")){
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075)
d['K'] = distance
}else if(d['A'].contains("烟台")){
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616)
d['K'] = distance
}
}
}, 1)
}
def addDistrict(){
def path = "D:\\数据\\入库写字楼.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def city = d['A']
def region = d['B']
def name = d['C']
def formatAddress = null
def address = d['D']
if(address){
if(d['G']){
formatAddress = AddressUtil.formatAddress(city, address)
}
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
def key = city+"##"+region+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","office")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
}
}
},0)
}
}
@@ -1,22 +1,20 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
@BatchOwner(Watcher.GaoJuan)
class PoiNineSplitBatch extends BaseBatch{
class PoiNineSplitBatch{
static main(args) {
runBatch(new PoiNineSplitBatch(), args)
def task = new PoiNineSplitBatch()
task.run()
}
@Override
void process(Object args) {
run()
}
// @Override
// void process(Object args) {
// run()
// }
def run(){
def i = 0;
while(i<360) {
@@ -1,105 +1,58 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.math.RandomUtils
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
def city = "昌江"
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
def date = d['B']
def start = DateUtils.getFirstDayOfMonth(date)
def end = DateUtils.getLastDayOfMonth(date)
def weekends = DateUtils.getWeekendBetweenDates(start, end)
for(def weekend : weekends){
def avg_price = d['I']
def val = random(-200,200)
def new_price = (int)(avg_price*val/10000 + avg_price)
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
def o = new BasicDBObject()
o.put("avg_price", new_price)
o.put("c_date" , new Date())
o.put("city", city)
o.put("m_date" , new Date())
o.put("weekend", weekend)
o.put("cat","城市")
o.put("moving_average" , (int)avg_price)
if(!last_price){
o.put("weekend_markup" , 0)
}else{
def weekend_markup = (int)((new_price-last_price)*100/last_price)
o.put("weekend_markup" , weekend_markup)
}
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
},1)
public static final int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
@@ -0,0 +1,134 @@
package com.fangjia.script.gj.zhongshu
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
class DistrictArrountOffice {
static main(args) {
def task = new DistrictArrountOffice()
task.run()
}
def run(){
def dataMap = ["北京":"116.481075,39.996743", "烟台":"121.438616,37.459645"]
def rets = []
rets << ["城市","区域","名称","地址","经度","纬度","写字楼等级","租金(起)","租金(止)", "价格来源"]
dataMap.each {k,v->
def locations = v.split(",")
def arroundOffices = CommonDatautils.getSeaweedArroundByCat(k, locations[0], locations[1], "office")
println k +" "+arroundOffices.size()
if(arroundOffices){
def index = 0
for(def o : arroundOffices){
index++
def priceMap = findEsOffice(o.city, o.region, o.name)
if(!priceMap){
priceMap = findEsHouse(o, "20231112")
}
def start_price = null
def end_price = null
def source = null
if(priceMap){
if(priceMap.avg_start_price){
start_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_start_price)
}
if(priceMap.avg_end_price){
end_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_end_price)
}
source = priceMap.source
}
def lng = DataHandleUtils.getDoubleRound(6, o.lng/10000000000.0)
def lat = DataHandleUtils.getDoubleRound(6, o.lat/10000000000.0)
def row = [o.city, o.region, o.name, o.address, lng, lat, o.bed_kind, start_price, end_price, source]
rets << row
println index +" "+ row
}
}
}
ExcelUtils.write("d:\\数据\\写字楼测试-1118.xlsx", rets, "数据")
}
def findEsOffice(city, region, name){
def map = [:]
def start_rent_prices = []
def end_rent_prices = []
def rent_prices = []
ElasticUtils.executeSqlQuery("select * from office where city='${city}' and region='${region}' and name='${name}'", { r->
if(r.start_rent_price){
start_rent_prices << r.start_rent_price
}
if(r.end_rent_price){
end_rent_prices << r.end_rent_price
}
if(r.rent_price){
rent_prices << r.rent_price
}
})
def avg_start_price = null
if(start_rent_prices){
avg_start_price = start_rent_prices.sum()/start_rent_prices.size()
}
def avg_end_price = null
if(end_rent_prices){
avg_end_price = end_rent_prices.sum()/end_rent_prices.size()
}
if(!avg_start_price && !avg_end_price){
if(rent_prices){
if(rent_prices.size()>1){
rent_prices = rent_prices.sort()
avg_start_price = rent_prices[0]
avg_end_price = rent_prices[rent_prices.size()-1]
}else{
avg_start_price = rent_prices[0] - rent_prices[0]*0.2
avg_end_price = rent_prices[0] + rent_prices[0]*0.2
}
}
}
if(avg_start_price){
map.put("avg_start_price", avg_start_price)
}
if(avg_end_price){
map.put("avg_end_price", avg_end_price)
}
if(map){
map.source = "非房源"
}
return map
}
def findEsHouse(o, startDate){
def rent_prices = CommonDatautils.findHousePriceByDate(o.city, o.region, o.name, o.alias, "office_rent", startDate)
def avg_start_price = null
def avg_end_price = null
def map = [:]
if(rent_prices){
if(rent_prices.size()>1){
rent_prices = rent_prices.sort()
avg_start_price = rent_prices[0]
avg_end_price = rent_prices[rent_prices.size()-1]
}else{
avg_start_price = rent_prices[0] - rent_prices[0]*0.2
avg_end_price = rent_prices[0] + rent_prices[0]*0.2
}
}
if(avg_start_price){
map.put("avg_start_price", avg_start_price)
}
if(avg_end_price){
map.put("avg_end_price", avg_end_price)
}
if(map){
map.source = "房源"
}
return map
}
}