Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+2
-3
@@ -1,15 +1,14 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+5
-9
@@ -10,12 +10,10 @@ import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
@@ -23,11 +21,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
@@ -71,11 +67,11 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
|
||||
+258
-2
@@ -2656,12 +2656,12 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrices(city, region, name, estateType, startDate, endDate){
|
||||
public static def queryDistrictPrices(city, region, name, startDate, endDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("estate_type", estateType);
|
||||
// queryCondition.put("estate_type", estateType);
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
def prices = [];
|
||||
@@ -4107,6 +4107,150 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def findHouseCountByDate(def city, def region, def name, def alias, def cat, def startDate){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("region", region))
|
||||
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
|
||||
|
||||
if(name || alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
static def findHousePriceByDate(def city, def region, def name, def alias, def cat, def startDate){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("region", region))
|
||||
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
|
||||
|
||||
if(name || alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
def prices = []
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
if(map.day_rent_price){
|
||||
prices << map.day_rent_price
|
||||
}
|
||||
}
|
||||
}
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
static def findNewPoiDatas(city){
|
||||
String sql = "select cityname,adname,name,address,location from poi_amap_2024 where (cityname='烟台市' or cityname='北京市') and typecode=120201"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
index++
|
||||
println index
|
||||
rets << [map.cityname, map.adname, map.name, map.address, map.location.x, map.location.y]
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
static def findAllNewPoiDatas(city){
|
||||
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
static def findOneThirdNetWorkOffice(city, region, name){
|
||||
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
|
||||
def map = [:]
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql, { m->
|
||||
if(m.location && !map){
|
||||
map = ["lng":m.location.x, "lat":m.location.y]
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
static def findThirdNetWorkOffice(){
|
||||
String sql = "select city,region,name,address,location from office where city='烟台' or city='北京'"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
index++
|
||||
println index
|
||||
if(map.location){
|
||||
rets << [map.city, map.region, map.name, map.address, map.location.lon, map.location.lat]
|
||||
}else{
|
||||
rets << [map.city, map.region, map.name, map.address, "", ""]
|
||||
}
|
||||
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
@@ -4773,6 +4917,21 @@ class CommonDatautils {
|
||||
return cityMap
|
||||
}
|
||||
|
||||
static def findFullNameCitys(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","城市")
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("province", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
|
||||
def cityList = []
|
||||
for(def d : dbcursor){
|
||||
cityList << d
|
||||
}
|
||||
return cityList
|
||||
}
|
||||
|
||||
static def queryDistNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
@@ -4991,6 +5150,34 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryFullRegionTownByCity(city, fields){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
for(def key : fields){
|
||||
keys.put(key,1)
|
||||
}
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.district){
|
||||
def list = townMap.get(d.district)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.district, list)
|
||||
}
|
||||
}
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
@@ -5259,4 +5446,73 @@ class CommonDatautils {
|
||||
return STATS_INDEX_COLL.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
static def findAllThirdPrice(city, region, name, statDate){
|
||||
def query = new BasicDBObject()
|
||||
query.append("category", "district")
|
||||
query.append("city", city)
|
||||
query.append("region", region)
|
||||
query.append("name", name)
|
||||
query.append("s_date", new BasicDBObject(QueryOperators.GTE, statDate))
|
||||
def dbCursor = thirdpartColl.find(query).sort(new BasicDBObject("s_date",-1))
|
||||
def retMap = [:]
|
||||
for(def d : dbCursor){
|
||||
|
||||
if(!retMap.get("xingfuli") && d.get("xingfuli_esf_price")){
|
||||
retMap.put("xingfuli", d.get("xingfuli_esf_price"))
|
||||
}
|
||||
if(!retMap.get("anjuke") && d.get("anjuke_esf_price")){
|
||||
retMap.put("anjuke", d.get("anjuke_esf_price"))
|
||||
}
|
||||
|
||||
if(!retMap.get("soufang") && d.get("soufang_esf_price")){
|
||||
retMap.put("soufang", d.get("soufang_esf_price"))
|
||||
}
|
||||
if(!retMap.get("lianjia") && d.get("lianjia_esf_price")){
|
||||
retMap.put("lianjia", d.get("lianjia_esf_price"))
|
||||
}
|
||||
if(!retMap.get("lianjia") && d.get("beike_esf_price")){
|
||||
retMap.put("lianjia", d.get("beike_esf_price"))
|
||||
}
|
||||
|
||||
if(!retMap.get("cityhouse") && d.get("cityhouse_esf_price")){
|
||||
retMap.put("cityhouse", d.get("cityhouse_esf_price"))
|
||||
}
|
||||
if(!retMap.get("jiwu") && d.get("jiwu_esf_price")){//使用吉屋替换房探 待抓取
|
||||
retMap.put("jiwu", d.get("jiwu_esf_price"))
|
||||
}
|
||||
|
||||
if(!retMap.get("house365") && d.get("365house_esf_price")){//使用淘房365替换房多多
|
||||
retMap.put("house365", d.get("365house_esf_price"))
|
||||
}
|
||||
|
||||
if(!retMap.get("maitian") && d.get("maitian_esf_price")){
|
||||
retMap.put("maitian", d.get("maitian_esf_price"))
|
||||
}
|
||||
if(!retMap.get("tc_58") && d.get("58tc_esf_price")){
|
||||
retMap.put("tc_58", d.get("58tc_esf_price"))
|
||||
}
|
||||
}
|
||||
|
||||
return retMap;
|
||||
}
|
||||
|
||||
static getSeaweedArroundByCat(city, lng, lat, cat) {
|
||||
try {
|
||||
def query = new BasicDBObject()
|
||||
query.append("cat", cat)
|
||||
query.append("city", city)
|
||||
query.append("visible", 0)
|
||||
DBObject sdq = new BasicDBObject();
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
|
||||
query.append('$sd', sdq)
|
||||
return DbUtils.seaweed.find(query).toList()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-5
@@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
|
||||
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.MongoClient
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
class AddCityRegionPrice {
|
||||
|
||||
@@ -24,7 +22,7 @@ class AddCityRegionPrice {
|
||||
static main(args) {
|
||||
def task = new AddCityRegionPrice()
|
||||
// task.addRegionPrice()
|
||||
task.addCityprice()
|
||||
// task.addCityprice()
|
||||
// task.addDistrictprice()
|
||||
// task.addCityRentPrice()
|
||||
// task.addRegionRentPrice()
|
||||
@@ -32,7 +30,7 @@ class AddCityRegionPrice {
|
||||
|
||||
// task.updateRegionRentData()
|
||||
// task.updateDealPriceAreaType()
|
||||
// task.checkCityImport()
|
||||
task.checkCityImport()
|
||||
}
|
||||
|
||||
def addRegionPrice(){
|
||||
@@ -255,7 +253,7 @@ class AddCityRegionPrice {
|
||||
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
|
||||
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
|
||||
if(startObj && endObj){
|
||||
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
|
||||
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410")
|
||||
months.each {m->
|
||||
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
|
||||
if(!obj){
|
||||
|
||||
+11
-7
@@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
|
||||
@@ -54,8 +57,8 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
//
|
||||
// }
|
||||
|
||||
// checkRangeCount()
|
||||
synch100()
|
||||
checkRangeCount()
|
||||
// synch100()
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
@@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{
|
||||
|
||||
|
||||
def checkRangeCount(){
|
||||
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||
// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||
|
||||
def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","乡镇街道")).batchSize(100)
|
||||
def count = 0
|
||||
def index =0
|
||||
for(def d : dbcursor){
|
||||
index ++
|
||||
if(d.location){
|
||||
def coordinates = d.location.coordinates
|
||||
if(d.shape){
|
||||
def coordinates = d.shape.coordinates
|
||||
if(coordinates.size()>1){
|
||||
count++
|
||||
println count +" "+ d.city +" "+d.name
|
||||
println count +" "+ d.city +" "+d.street
|
||||
}
|
||||
}else{
|
||||
println d.city + " "+d.name
|
||||
println d.city + " "+d.street
|
||||
}
|
||||
|
||||
if(index%1000==0){
|
||||
|
||||
+50
-22
@@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 小区自动入库(新增小区,每月运行一次)
|
||||
*/
|
||||
@@ -36,22 +35,39 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = ["上海"]//CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||
for(def c : citys){
|
||||
def noMatchDatas = findNoMatchResult(c)
|
||||
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
def rets = checkDatas(noMatchDatas.keySet())
|
||||
list.addAll(rets)
|
||||
}
|
||||
// //查找第三方匹配没有匹配上的小区
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def list = []
|
||||
// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
|
||||
// for(def c : citys){
|
||||
// def noMatchDatas = findNoMatchResult(c)
|
||||
// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
// def rets = checkDatas(noMatchDatas.keySet())
|
||||
// list.addAll(rets)
|
||||
// }
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
|
||||
fileName = "D:\\数据\\add_dist_info_1.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
// ExcelUtils.write(fileName, list, "待入库数据")
|
||||
|
||||
|
||||
check(fileName)
|
||||
}
|
||||
|
||||
def check(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
|
||||
if(dists){
|
||||
d['J'] = "库中已存在"
|
||||
}else{
|
||||
dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
|
||||
if(dists){
|
||||
d['J'] = "库中别名中存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkDatas(keysets){
|
||||
@@ -72,11 +88,23 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
//检测有没有房源
|
||||
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
|
||||
//检查百度地图有没有
|
||||
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||
def baidu_obj = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||
def baidu_count = 0
|
||||
def baidu_type = null
|
||||
if(baidu_obj){
|
||||
baidu_count = 1
|
||||
baidu_type = baidu_obj.std_tag
|
||||
}
|
||||
//检查高德地图有没有
|
||||
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
def gaode_count = 0
|
||||
def gaode_type = null
|
||||
if(gaode_obj){
|
||||
gaode_count = 1
|
||||
gaode_type = gaode_obj.type
|
||||
}
|
||||
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -123,18 +151,18 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
if(fullDistName){
|
||||
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
|
||||
if(obj){
|
||||
return 1
|
||||
return obj
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
return null
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name && d.type.contains("住宅") && d.shape){
|
||||
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
@@ -228,7 +256,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
def count = BAIDU_POI_COLL.count(q)
|
||||
return count;
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
+37
-6
@@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.joda.time.Instant
|
||||
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DevProcess {
|
||||
@@ -64,6 +58,7 @@ class DevProcess {
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
task.synchDevTo100()
|
||||
// task.statShape()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
@@ -87,6 +82,8 @@ class DevProcess {
|
||||
// task.matchDevShape()//匹配开发区围栏
|
||||
|
||||
// task.addQiDevResult() //获取齐梁数据结果
|
||||
|
||||
// task.updateDevShape()//修改开发区结构
|
||||
}
|
||||
|
||||
|
||||
@@ -794,6 +791,18 @@ class DevProcess {
|
||||
development_zone_CONN.insert(rets)
|
||||
}
|
||||
|
||||
def statShape(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true)))
|
||||
def index = 0
|
||||
for(def r : dbcursor){
|
||||
index ++
|
||||
def coordinates = r.shape.coordinates
|
||||
println index+" "+r.name+" "+coordinates.size()
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
|
||||
@@ -1148,4 +1157,26 @@ class DevProcess {
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateDevShape(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("shape", new BasicDBObject('$exists', true))
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
if(d.name!="福州长乐国际机场综合保税区"){
|
||||
def udata = new BasicDBObject()
|
||||
if(d.shape.coordinates){
|
||||
d.shape.coordinates = [d.shape.coordinates]
|
||||
udata.put("shape", d.shape)
|
||||
}
|
||||
if(d.bd_shape && d.bd_shape.coordinates){
|
||||
d.bd_shape.coordinates = [d.bd_shape.coordinates]
|
||||
udata.put("bd_shape", d.bd_shape)
|
||||
}
|
||||
def updateQuery = new BasicDBObject("_id",d._id)
|
||||
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-3
@@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -33,7 +32,7 @@ class DevUpdateTestBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
//
|
||||
@@ -168,7 +167,7 @@ class DevUpdateTestBatch {
|
||||
println d['A'] +" ²»´æÔÚ"
|
||||
}
|
||||
}
|
||||
},2)
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+126
-1
@@ -1,17 +1,28 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.SerializationUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 查找重复小区
|
||||
*/
|
||||
class DistrictRepeatBatch {
|
||||
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictRepeatBatch()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.resultProcess()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -65,4 +76,118 @@ class DistrictRepeatBatch {
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
def resultProcess(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d->
|
||||
if(d["O"] && !d["O"].contains("不处理") && !d['R']){
|
||||
if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){
|
||||
def updteId = d['H']
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(updteId))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
if(d['O']=="K列改为商住" ){
|
||||
udata.put("estate_type2", "商住")
|
||||
}else{
|
||||
udata.put("estate_type2", "别墅")
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["I"], d["J"], d["K"], "update")
|
||||
d["R"] = "库中已处理"
|
||||
}
|
||||
else if(d["O"] == "都删"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["B"], d["C"], d["D"], "remove")
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["I"], d["J"], d["K"], "remove")
|
||||
d["R"] = "库中已处理"
|
||||
}
|
||||
else if(d["O"]=="删除K列小区"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
|
||||
saveOptLog(d["I"], d["J"], d["K"], "remove")
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
if(d['C']==d['J']){
|
||||
def alias = []
|
||||
def addr_alias = []
|
||||
if(d['D']!=d['K']){
|
||||
def s1 = d['D'].replace(d['K'], "")
|
||||
def s2 = d['K'].replace(d['D'], "")
|
||||
def addNameAlias = true
|
||||
if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){
|
||||
addNameAlias = false
|
||||
}
|
||||
if(addNameAlias){
|
||||
|
||||
if(obj.alias){
|
||||
alias = SerializationUtils.clone(obj.alias)
|
||||
}
|
||||
if(!alias.contains(d['K'])){
|
||||
alias << d['K']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def address = null
|
||||
if(d['M'] && d['F']!=d['M']){
|
||||
def formatAddress = AddressUtil.formatAddress(d['A'], d['M'])
|
||||
if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){
|
||||
if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){
|
||||
if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){
|
||||
address = formatAddress
|
||||
}else{
|
||||
if(obj.addr_alias){
|
||||
addr_alias = SerializationUtils.clone(obj.addr_alias)
|
||||
}
|
||||
if(!addr_alias.contains(formatAddress)){
|
||||
addr_alias << formatAddress
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){
|
||||
def udata2 = new BasicDBObject()
|
||||
if(alias){
|
||||
udata2.put("alias", alias)
|
||||
}
|
||||
if(addr_alias){
|
||||
udata2.put("addr_alias", addr_alias)
|
||||
}
|
||||
if(address){
|
||||
udata2.put("address", address)
|
||||
}
|
||||
udata2.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2))
|
||||
}
|
||||
}
|
||||
|
||||
d["R"] = "库中已处理"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, type){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","district");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", type)
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
/**
|
||||
* 新版poi处理入库(第二批)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
public static def TAGS = []
|
||||
|
||||
static main(args) {
|
||||
runBatch(new NewGaoDePoiBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
if (CommonDatautils.isWindow()) {
|
||||
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
|
||||
}else{
|
||||
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
|
||||
}
|
||||
|
||||
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
for(def cityObj : cityList){
|
||||
// if(cityObj.city == "胡杨河市"){
|
||||
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
|
||||
println cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def rows = running(cityObj.city, results)
|
||||
save(rows)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
def getAllPoiType(filePath){
|
||||
def poiTypes = ExcelUtils.read(filePath, 0, 1)
|
||||
def types = []
|
||||
for(def poi : poiTypes){
|
||||
types << poi['A']
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
def running(city,results){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
|
||||
def mzbRegionMap = regionFullNameMap.get(city)
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets.addAll(createData(map, mzbRegionMap, townMap))
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def getStandardRegion(mzbRegionMap, district){
|
||||
def tjjObj = mzbRegionMap.get(district)
|
||||
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name == district){
|
||||
return district
|
||||
}
|
||||
def updateRegion = null
|
||||
if(tjjObj){
|
||||
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=district){
|
||||
updateRegion = tjjObj.mzb_district_name
|
||||
}
|
||||
}else{
|
||||
def set = mzbRegionMap.keySet()
|
||||
for(def key : set){
|
||||
def tempObj = mzbRegionMap.get(key)
|
||||
def alias = tempObj.alias
|
||||
if(alias.contains(district) || district==tempObj.mzb_district_name){
|
||||
updateRegion = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return updateRegion
|
||||
}
|
||||
|
||||
def createData(map, mzbRegionMap, townMap){
|
||||
def saveObj = [:]
|
||||
saveObj.province = map.province
|
||||
saveObj.city = map.city
|
||||
def newDistrict = getStandardRegion(mzbRegionMap, map.district)
|
||||
if(newDistrict){
|
||||
saveObj.district = newDistrict
|
||||
}else{
|
||||
println map.city +" "+map.district +" 格式化不出区域"
|
||||
}
|
||||
|
||||
saveObj.name = map.name
|
||||
|
||||
if(map.cat){
|
||||
saveObj = catWash(saveObj, map)
|
||||
}
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, map.lat)
|
||||
saveObj.location = location
|
||||
saveObj.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
saveObj.version = version
|
||||
saveObj.data_id = map.map_id
|
||||
saveObj.data_source = "高德地图"
|
||||
saveObj.data_source_year = "2024-11"
|
||||
|
||||
if(saveObj.cat && TAGS.contains(saveObj.cat)){
|
||||
saveObj.status = 0
|
||||
}else{
|
||||
saveObj.status = 1
|
||||
}
|
||||
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if(location){
|
||||
saveObj = addTown(saveObj, towns, location)
|
||||
}
|
||||
|
||||
if(map.address){
|
||||
saveObj = addressWash(saveObj, map, mzbRegionMap, towns)
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def catWash(saveObj, map){
|
||||
def arrs = map.cat.split(";|:")
|
||||
if(arrs.size()>=1){
|
||||
saveObj.type = arrs[0]
|
||||
}
|
||||
if(arrs.size()>=2){
|
||||
saveObj.secondary_type = arrs[1]
|
||||
}else{
|
||||
saveObj.secondary_type = saveObj.type
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
def smallstr = arrs[2]
|
||||
def smallArr = null
|
||||
if(smallstr.contains("|")){
|
||||
smallArr = smallstr.split("|")
|
||||
}
|
||||
if(smallArr && smallArr.size()>=1){
|
||||
saveObj.small_type = smallArr[0]
|
||||
}else{
|
||||
saveObj.small_type = smallstr
|
||||
}
|
||||
}else{
|
||||
saveObj.small_type = saveObj.secondary_type
|
||||
}
|
||||
|
||||
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def addTown(saveObj, towns, location){
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
if(town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(map.address && !saveObj.township){
|
||||
// for(def town : towns){
|
||||
// if(map.address.contains(town.street)){
|
||||
// saveObj.township = town.street
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
|
||||
// println map.address +" 含街道,镇,乡"
|
||||
// }
|
||||
// }
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def addressWash(saveObj, map, mzbRegionMap, towns){
|
||||
if(map.address.length()<3){
|
||||
saveObj.address = ""
|
||||
}else{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
while(i<5){
|
||||
if(address && address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(map.province.toString().length())
|
||||
}
|
||||
if(address && address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(map.city.toString().length())
|
||||
}
|
||||
|
||||
if(address && saveObj.district){
|
||||
def tjjObj = mzbRegionMap.get(saveObj.district)
|
||||
if(tjjObj){
|
||||
def alias = null
|
||||
if(tjjObj.alias){
|
||||
alias = new CopyOnWriteArrayList<>(tjjObj.alias);
|
||||
}else{
|
||||
alias = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
if(!alias.contains(saveObj.district)){
|
||||
alias << saveObj.district
|
||||
}
|
||||
if(alias.size()>=2){
|
||||
alias = alias.sort(){ a, b ->
|
||||
b.length().compareTo(a.length());
|
||||
}
|
||||
}
|
||||
|
||||
for(def regionName : alias){
|
||||
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
|
||||
address = address.substring(regionName.toString().length())
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(address && address.toString().startsWith("市辖区")){
|
||||
address = address.toString().substring(3)
|
||||
}
|
||||
|
||||
if(address && towns){
|
||||
for(def town : towns){
|
||||
def townAlias = null
|
||||
if(town.alias){
|
||||
townAlias = new CopyOnWriteArrayList<>(town.alias);
|
||||
}else{
|
||||
townAlias = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
if(!townAlias.contains(town.street)){
|
||||
townAlias << town.street
|
||||
}
|
||||
if(townAlias.size()>=2){
|
||||
townAlias = townAlias.sort(){ a, b ->
|
||||
b.length().compareTo(a.length());
|
||||
}
|
||||
}
|
||||
|
||||
for(def townName : townAlias){
|
||||
if(address.startsWith(townName)){
|
||||
address = address.substring(townName.toString().length())
|
||||
// if(!saveObj.township){
|
||||
// saveObj.township = town.street
|
||||
// }
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
if(address && address.startsWith("(") && address.endsWith(")")){
|
||||
saveObj.address = address.substring(1, address.length()-1)
|
||||
}else{
|
||||
saveObj.address = address
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 2000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
|
||||
rows.each{map->
|
||||
// def map = rows.get(key)
|
||||
|
||||
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class NewPoiData {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new NewPoiData()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
// export()
|
||||
// exportNetWorkOffice()
|
||||
// matchByCity()
|
||||
// noMatchData()
|
||||
calcuteDistance()
|
||||
}
|
||||
|
||||
def noMatchData(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\未匹配写字楼2.xlsx", { d ->
|
||||
if(d['H']!='地图'){
|
||||
def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C'])
|
||||
if(map){
|
||||
d['E'] = map.lng
|
||||
d['F'] = map.lat
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
def matchByCity(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\入库写字楼.xlsx",{d->
|
||||
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false)
|
||||
def formatAddress = null
|
||||
if(d['D']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
d['G'] = formatAddress
|
||||
}
|
||||
if(!jsonObject && formatAddress){
|
||||
if(DataUtil.isPreciseAddress(formatAddress)){
|
||||
def address = d['A']
|
||||
if(d['B']){
|
||||
address = address + d['B']
|
||||
}
|
||||
address = address + formatAddress
|
||||
jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false)
|
||||
}
|
||||
}
|
||||
if(jsonObject) {
|
||||
d['H'] = jsonObject.get("id")
|
||||
d['I'] = jsonObject.get("city")
|
||||
d['J'] = jsonObject.get("district")
|
||||
d['K'] = jsonObject.get("name")
|
||||
d['L'] = jsonObject.get("propertyType")
|
||||
d['M'] = jsonObject.get("credit")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def export(){
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
|
||||
rets.addAll(CommonDatautils.findNewPoiDatas())
|
||||
ExcelUtils.write("d:\\数据\\poi写字楼.xlsx", rets, "写字楼")
|
||||
}
|
||||
|
||||
def exportNetWorkOffice(){
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
|
||||
rets.addAll(CommonDatautils.findThirdNetWorkOffice())
|
||||
ExcelUtils.write("d:\\数据\\poi写字楼-网站.xlsx", rets, "写字楼")
|
||||
}
|
||||
|
||||
def calcuteDistance(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\写字楼测试-1118-新增.xlsx", { d ->
|
||||
if(d['F']){
|
||||
if(d['A'].contains("北京")){
|
||||
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075)
|
||||
d['K'] = distance
|
||||
}else if(d['A'].contains("烟台")){
|
||||
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616)
|
||||
d['K'] = distance
|
||||
}
|
||||
}
|
||||
}, 1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\入库写字楼.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def name = d['C']
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['D']
|
||||
if(address){
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
}
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def key = city+"##"+region+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","office")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class PoiNineSplitBatch{
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiNineSplitBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
// @Override
|
||||
// void process(Object args) {
|
||||
// run()
|
||||
// }
|
||||
def run(){
|
||||
def i = 0;
|
||||
while(i<360) {
|
||||
def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\new_poi\\new_poi_address_split_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if (d['C'] && !d['G']) {
|
||||
println d['C'] +" "+ d['E']
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E'])
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
d['G'] = jsonObject.get("street")
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
d['H'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
|
||||
d['I'] = jsonObject.get("township")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
println fileName +" 九级拆分完成"
|
||||
}else{
|
||||
println fileName +" 文件不存在"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
+48
-13
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
@@ -41,12 +42,10 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
|
||||
def index = 0
|
||||
def citys = ["ÅÊÖ¦»¨"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
index ++
|
||||
// if(index<32){
|
||||
// continue
|
||||
// }
|
||||
|
||||
if(runCitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
@@ -57,19 +56,32 @@ class RegionStandardBatch extends BaseBatch{
|
||||
}
|
||||
if(fullCityName){
|
||||
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
// def rets = findEsPoiByCity(fullCityName)
|
||||
def rets = readCSV(fileName)
|
||||
println city +" "+rets.size()
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
|
||||
println "没找区域:"+city
|
||||
}
|
||||
}else{
|
||||
println "没找区域:"+city
|
||||
}
|
||||
}else{
|
||||
println "ûÕÒÇøÓò£º"+city
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
@@ -365,6 +377,29 @@ class RegionStandardBatch extends BaseBatch{
|
||||
//
|
||||
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||
// }
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println filename+" 文件异常"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+49
-4
@@ -1,15 +1,14 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class TempExportData {
|
||||
|
||||
@@ -17,7 +16,10 @@ class TempExportData {
|
||||
|
||||
static main(args) {
|
||||
def task = new TempExportData()
|
||||
task.run()
|
||||
// task.run()
|
||||
def rets = task.fzHouseStat()
|
||||
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
|
||||
println "文件已生成"
|
||||
}
|
||||
|
||||
def run() {
|
||||
@@ -75,4 +77,47 @@ class TempExportData {
|
||||
println(city +" "+rets.size())
|
||||
return rets;
|
||||
}
|
||||
|
||||
def fzHouseStat(){
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "alias","estate_type2"])
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def total = districts.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
rets << ["城市","区域","名称","上月价格","3年房源量","2年房源量", "2年第三方家数"]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def o = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def prices = CommonDatautils.queryDistrictPrices(o.city, o.region, o.name, "2024-10-01", "2024-10-31")
|
||||
def avg_price = 0
|
||||
if(prices){
|
||||
avg_price = prices[0]
|
||||
}
|
||||
|
||||
def count1 = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20211115)
|
||||
def count2 = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20221115)
|
||||
|
||||
|
||||
def thirdMap = CommonDatautils.findAllThirdPrice(o.city, o.region, o.name,202211)
|
||||
def thirdCount = 0
|
||||
if(thirdMap){
|
||||
thirdCount = thirdMap.size()
|
||||
}
|
||||
rets << [o.city, o.region, o.name, avg_price, count1, count2, thirdCount]
|
||||
println addCount.get() + " / "+ total
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
}
|
||||
+42
-89
@@ -1,105 +1,58 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
def city = "昌江"
|
||||
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
|
||||
def date = d['B']
|
||||
def start = DateUtils.getFirstDayOfMonth(date)
|
||||
def end = DateUtils.getLastDayOfMonth(date)
|
||||
def weekends = DateUtils.getWeekendBetweenDates(start, end)
|
||||
for(def weekend : weekends){
|
||||
def avg_price = d['I']
|
||||
def val = random(-200,200)
|
||||
def new_price = (int)(avg_price*val/10000 + avg_price)
|
||||
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
|
||||
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
def o = new BasicDBObject()
|
||||
o.put("avg_price", new_price)
|
||||
o.put("c_date" , new Date())
|
||||
o.put("city", city)
|
||||
o.put("m_date" , new Date())
|
||||
o.put("weekend", weekend)
|
||||
o.put("cat","城市")
|
||||
o.put("moving_average" , (int)avg_price)
|
||||
if(!last_price){
|
||||
o.put("weekend_markup" , 0)
|
||||
}else{
|
||||
def weekend_markup = (int)((new_price-last_price)*100/last_price)
|
||||
o.put("weekend_markup" , weekend_markup)
|
||||
}
|
||||
|
||||
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
|
||||
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
},1)
|
||||
|
||||
|
||||
public static final int random(int i1, int i2) {
|
||||
int max = Math.max(i1, i2);
|
||||
int min = Math.min(i1, i2);
|
||||
return min + RandomUtils.nextInt(max - min + 1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+8
-7
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -10,17 +10,15 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThardParthAnalyBatch extends BaseBatch{
|
||||
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static DBCollection data_process_coll = CrawalDBUtil.getDataProcessCol()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThardParthAnalyBatch(), args)
|
||||
@@ -119,11 +117,14 @@ class ThardParthAnalyBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def findMatchSuccessDistCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
// def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
// def dbcursor = thirdPartyColl.find(q)
|
||||
|
||||
def q = new BasicDBObject().append("s_date", sDate)
|
||||
def dbcursor = data_process_coll.find(q)
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
set << d.match_city+"##"+d.match_region+"##"+d.match_name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
+17
-7
@@ -4,7 +4,6 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
@@ -12,14 +11,11 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 第三方小区与我们小区映射关系表
|
||||
*/
|
||||
@@ -45,7 +41,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
// //将新增数据同步过来
|
||||
// addNemCommunity()
|
||||
// //接口进行匹配
|
||||
//// //½Ó¿Ú½øÐÐÆ¥Åä
|
||||
match()
|
||||
|
||||
addOldMatchResult()
|
||||
@@ -53,7 +49,12 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
|
||||
def match(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 206){
|
||||
continue
|
||||
}
|
||||
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
|
||||
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
matchByCity(c, noMatchDatas)
|
||||
@@ -181,6 +182,10 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("source", 1)
|
||||
|
||||
for(def c : citys){
|
||||
def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList()
|
||||
@@ -200,8 +205,13 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString())
|
||||
def obj = dataProcessColl.findOne(query)
|
||||
//.append("s_date", curDate)
|
||||
def query = new BasicDBObject().append("city", city).append("region",d.district).append("name", d.name).append("source",d.source)
|
||||
def objs = dataProcessColl.find(query).sort(new BasicDBObject("s_date",-1)).limit(1)
|
||||
def obj = null
|
||||
if(objs){
|
||||
obj = objs[0]
|
||||
}
|
||||
if(obj) {
|
||||
def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district")
|
||||
def udata = new BasicDBObject()
|
||||
|
||||
+26
-6
@@ -24,6 +24,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
def updateList = []
|
||||
def addAliasList = []
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownStandardBatch(), args)
|
||||
@@ -66,7 +67,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def townStandardByPoi(){
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
|
||||
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
|
||||
//浙江,江苏,安徽
|
||||
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
@@ -78,7 +82,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\${fullCityName}.csv"
|
||||
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
@@ -88,7 +92,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
|
||||
def set = rMap.keySet()
|
||||
// def rets = findEsPoiByCity(fullCityName)
|
||||
def rets = readPoiCsv(fullCityName)
|
||||
def rets = readPoiCsv(fileName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||
@@ -96,12 +100,16 @@ class TownStandardBatch extends BaseBatch{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println fileName+" 文件没找到"
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
|
||||
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
def results = []
|
||||
@@ -194,6 +202,18 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
break
|
||||
}
|
||||
// else{
|
||||
// if(town.street.contains(d.township.replace("镇","").replace("街道","").replace("乡",""))){
|
||||
// updateTown = town.street
|
||||
// opt = false
|
||||
// def addObj = [town.city, town.district, town.street, d.township]
|
||||
// if(!addAliasList.contains(addObj)){
|
||||
// addAliasList << addObj
|
||||
// }
|
||||
//
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
if(opt){
|
||||
@@ -206,11 +226,11 @@ class TownStandardBatch extends BaseBatch{
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
results << m
|
||||
// results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
@@ -256,7 +276,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
println("批量更新中有失败的操作:")
|
||||
bulkResponse.buildFailureMessage().each { println(it) }
|
||||
} else {
|
||||
println("批量更新成功!")
|
||||
// println("批量更新成功!")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import org.bouncycastle.crypto.engines.SM4Engine;
|
||||
import org.bouncycastle.crypto.modes.CBCBlockCipher;
|
||||
import org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher;
|
||||
import org.bouncycastle.crypto.params.KeyParameter;
|
||||
import org.bouncycastle.crypto.params.ParametersWithIV;
|
||||
import org.bouncycastle.util.encoders.Hex;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/10/9
|
||||
*/
|
||||
public class CryptoSm4Util {
|
||||
|
||||
/**
|
||||
* 默认密钥
|
||||
*/
|
||||
private static final String DEFAULT_SECRET_KEY = "A1B2C3D4E5F60789A1B2C3D4E5F60789";
|
||||
private static final String CHARSET = "UTF-8";
|
||||
|
||||
// String plaintext -> Base64-encoded String ciphertext
|
||||
public static String encrypt(String key, String plaintext) {
|
||||
try {
|
||||
byte[] keyBytes = Hex.decode(key);
|
||||
byte[] iv = generateIV();
|
||||
PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
|
||||
cipher.init(true, new ParametersWithIV(new KeyParameter(keyBytes), iv));
|
||||
byte[] input = plaintext.getBytes(CHARSET);
|
||||
byte[] encrypted = new byte[cipher.getOutputSize(input.length)];
|
||||
int length1 = cipher.processBytes(input, 0, input.length, encrypted, 0);
|
||||
int length2 = cipher.doFinal(encrypted, length1);
|
||||
byte[] finalEncrypted = new byte[length1 + length2];
|
||||
System.arraycopy(encrypted, 0, finalEncrypted, 0, finalEncrypted.length);
|
||||
return Hex.toHexString(iv) + Hex.toHexString(finalEncrypted);
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static byte[] generateIV() {
|
||||
byte[] iv = new byte[16];
|
||||
new SecureRandom().nextBytes(iv);
|
||||
return iv;
|
||||
}
|
||||
|
||||
// Base64-encoded String ciphertext -> String plaintext
|
||||
public static String decrypt(String key, String ciphertext) {
|
||||
try {
|
||||
byte[] keyBytes = Hex.decode(key);
|
||||
byte[] iv = Hex.decode(ciphertext.substring(0, 32));
|
||||
byte[] encryptedData = Hex.decode(ciphertext.substring(32));
|
||||
PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
|
||||
cipher.init(false, new ParametersWithIV(new KeyParameter(keyBytes), iv));
|
||||
byte[] decrypted = new byte[cipher.getOutputSize(encryptedData.length)];
|
||||
int length1 = cipher.processBytes(encryptedData, 0, encryptedData.length, decrypted, 0);
|
||||
int length2 = cipher.doFinal(decrypted, length1);
|
||||
byte[] finalDecrypted = new byte[length1 + length2];
|
||||
System.arraycopy(decrypted, 0, finalDecrypted, 0, finalDecrypted.length);
|
||||
return new String(finalDecrypted, CHARSET).trim();
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用默认密钥加密
|
||||
*
|
||||
* @param str 加密内容
|
||||
* @return 以String密文输入, String明文输出
|
||||
*/
|
||||
public static String encode(String str) {
|
||||
return encode(DEFAULT_SECRET_KEY, str);
|
||||
}
|
||||
|
||||
/**
|
||||
* 自定义密钥加密
|
||||
*
|
||||
* @param key 自定义密钥
|
||||
* @param str 加密的内容
|
||||
* @return String明文输入, String密文输出
|
||||
*/
|
||||
public static String encode(String key, String str) {
|
||||
return encrypt(key, str);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用默认密钥解密
|
||||
*
|
||||
* @param str 解密内容
|
||||
* @return 以String密文输入, String明文输出
|
||||
*/
|
||||
public static String decode(String str) {
|
||||
return decode(DEFAULT_SECRET_KEY, str);
|
||||
}
|
||||
|
||||
/**
|
||||
* 自定义密钥解密
|
||||
*
|
||||
* @param key 自定义密钥
|
||||
* @param str 加密内容
|
||||
* @return 以String密文输入, String明文输出
|
||||
*/
|
||||
public static String decode(String key, String str) {
|
||||
return decrypt(key, str);
|
||||
}
|
||||
|
||||
public static void main(def a){
|
||||
println encrypt( DEFAULT_SECRET_KEY, "测试")
|
||||
}
|
||||
}
|
||||
+21
-7
@@ -15,10 +15,14 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 厦门涉台园区数据同步
|
||||
*/
|
||||
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
|
||||
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
|
||||
|
||||
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
|
||||
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
|
||||
@@ -28,6 +32,8 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RelatedTaiWanBatch(), args);
|
||||
}
|
||||
@@ -37,14 +43,11 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
def day = new Date().getDate()
|
||||
if(day == 10){
|
||||
//createDatas() //企业库获取涉台园区企业
|
||||
// createDatas() //企业库获取涉台园区企业
|
||||
updateRangeByPoint() //更改企业所在自贸区
|
||||
export() //导出数据
|
||||
}else{
|
||||
exportEmptyFile()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -152,10 +155,18 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
}
|
||||
def allDatas = []
|
||||
allDatas.addAll(rows)
|
||||
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
|
||||
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
|
||||
|
||||
//原不加密之前
|
||||
// FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
|
||||
// XmBankFtpUtils.ftpUpload(fileName, curDate)
|
||||
|
||||
|
||||
def str1 = FileUtils.createFileStringByCustomSuffix(rows, fileName, "utf-8", "|@|","|+|");
|
||||
def str1_encryptedText = CryptoSm4Util.encrypt( CryptoSm4Util.DEFAULT_SECRET_KEY, str1)
|
||||
FileUtils.createEncryptFile(fileName, str1_encryptedText, false)
|
||||
XmBankFtpUtils.ftpUpload(fileName, curDate)
|
||||
|
||||
|
||||
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
|
||||
@@ -181,4 +192,7 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
package com.fangjia.script.gj.zhongshu
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class DistrictArrountOffice {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictArrountOffice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def dataMap = ["北京":"116.481075,39.996743", "烟台":"121.438616,37.459645"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","经度","纬度","写字楼等级","租金(起)","租金(止)", "价格来源"]
|
||||
dataMap.each {k,v->
|
||||
def locations = v.split(",")
|
||||
def arroundOffices = CommonDatautils.getSeaweedArroundByCat(k, locations[0], locations[1], "office")
|
||||
println k +" "+arroundOffices.size()
|
||||
if(arroundOffices){
|
||||
def index = 0
|
||||
for(def o : arroundOffices){
|
||||
index++
|
||||
def priceMap = findEsOffice(o.city, o.region, o.name)
|
||||
if(!priceMap){
|
||||
priceMap = findEsHouse(o, "20231112")
|
||||
}
|
||||
def start_price = null
|
||||
def end_price = null
|
||||
def source = null
|
||||
if(priceMap){
|
||||
if(priceMap.avg_start_price){
|
||||
start_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_start_price)
|
||||
}
|
||||
if(priceMap.avg_end_price){
|
||||
end_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_end_price)
|
||||
}
|
||||
source = priceMap.source
|
||||
}
|
||||
def lng = DataHandleUtils.getDoubleRound(6, o.lng/10000000000.0)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, o.lat/10000000000.0)
|
||||
def row = [o.city, o.region, o.name, o.address, lng, lat, o.bed_kind, start_price, end_price, source]
|
||||
rets << row
|
||||
println index +" "+ row
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\写字楼测试-1118.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
def findEsOffice(city, region, name){
|
||||
def map = [:]
|
||||
def start_rent_prices = []
|
||||
def end_rent_prices = []
|
||||
def rent_prices = []
|
||||
|
||||
ElasticUtils.executeSqlQuery("select * from office where city='${city}' and region='${region}' and name='${name}'", { r->
|
||||
if(r.start_rent_price){
|
||||
start_rent_prices << r.start_rent_price
|
||||
}
|
||||
if(r.end_rent_price){
|
||||
end_rent_prices << r.end_rent_price
|
||||
}
|
||||
if(r.rent_price){
|
||||
rent_prices << r.rent_price
|
||||
}
|
||||
|
||||
})
|
||||
def avg_start_price = null
|
||||
if(start_rent_prices){
|
||||
avg_start_price = start_rent_prices.sum()/start_rent_prices.size()
|
||||
}
|
||||
def avg_end_price = null
|
||||
if(end_rent_prices){
|
||||
avg_end_price = end_rent_prices.sum()/end_rent_prices.size()
|
||||
}
|
||||
|
||||
if(!avg_start_price && !avg_end_price){
|
||||
if(rent_prices){
|
||||
if(rent_prices.size()>1){
|
||||
rent_prices = rent_prices.sort()
|
||||
avg_start_price = rent_prices[0]
|
||||
avg_end_price = rent_prices[rent_prices.size()-1]
|
||||
}else{
|
||||
avg_start_price = rent_prices[0] - rent_prices[0]*0.2
|
||||
avg_end_price = rent_prices[0] + rent_prices[0]*0.2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(avg_start_price){
|
||||
map.put("avg_start_price", avg_start_price)
|
||||
}
|
||||
if(avg_end_price){
|
||||
map.put("avg_end_price", avg_end_price)
|
||||
}
|
||||
if(map){
|
||||
map.source = "非房源"
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findEsHouse(o, startDate){
|
||||
def rent_prices = CommonDatautils.findHousePriceByDate(o.city, o.region, o.name, o.alias, "office_rent", startDate)
|
||||
def avg_start_price = null
|
||||
def avg_end_price = null
|
||||
def map = [:]
|
||||
if(rent_prices){
|
||||
if(rent_prices.size()>1){
|
||||
rent_prices = rent_prices.sort()
|
||||
avg_start_price = rent_prices[0]
|
||||
avg_end_price = rent_prices[rent_prices.size()-1]
|
||||
}else{
|
||||
avg_start_price = rent_prices[0] - rent_prices[0]*0.2
|
||||
avg_end_price = rent_prices[0] + rent_prices[0]*0.2
|
||||
}
|
||||
}
|
||||
if(avg_start_price){
|
||||
map.put("avg_start_price", avg_start_price)
|
||||
}
|
||||
if(avg_end_price){
|
||||
map.put("avg_end_price", avg_end_price)
|
||||
}
|
||||
if(map){
|
||||
map.source = "房源"
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
+12
-25
@@ -7,21 +7,8 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.CommonUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DB
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang3.SerializationUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.apache.tools.ant.util.DateUtils
|
||||
@@ -31,9 +18,7 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 第三方数据标准化处理
|
||||
* 成功type说明
|
||||
@@ -689,14 +674,15 @@ class DataStandardBatch extends BaseBatch{
|
||||
def pre = (data.avg_price-price)*100/price
|
||||
if(pre>=-20 && pre<=20){
|
||||
return dbo;
|
||||
}else{
|
||||
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
|
||||
if(!ret){
|
||||
data.put("msg","·¿Ô´¼Û¸ñ¹ýÂË")
|
||||
THIRD_PRICE_FILTER_COLL.save(data)
|
||||
}
|
||||
return null
|
||||
}
|
||||
// else{
|
||||
// def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
|
||||
// if(!ret){
|
||||
// data.put("msg","房源价格过滤")
|
||||
// THIRD_PRICE_FILTER_COLL.save(data)
|
||||
// }
|
||||
// return null
|
||||
// }
|
||||
}
|
||||
}
|
||||
return dbo
|
||||
@@ -934,7 +920,8 @@ class DataStandardBatch extends BaseBatch{
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
||||
// qdata.put("city","上海")
|
||||
// qdata.put("name","°×½ð嫹¬")
|
||||
// qdata.put("name","绿地云天坊")
|
||||
// qdata.put("status", 3)
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
|
||||
Reference in New Issue
Block a user