Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2024-12-02 10:31:31 +08:00
23 changed files with 1549 additions and 216 deletions
+7
View File
@@ -530,6 +530,13 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15to18</artifactId>
<version>1.70</version> <!-- 确保使用最新版本 -->
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -1,15 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\2024年10月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -10,12 +10,10 @@ import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import org.bson.types.ObjectId
@@ -23,11 +21,9 @@ import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
@@ -71,11 +67,11 @@ class BigdataDealBatch extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -2656,12 +2656,12 @@ class CommonDatautils {
}
public static def queryDistrictPrices(city, region, name, estateType, startDate, endDate){
public static def queryDistrictPrices(city, region, name, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("estate_type", estateType);
// queryCondition.put("estate_type", estateType);
queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate)));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
def prices = [];
@@ -4107,6 +4107,150 @@ class CommonDatautils {
}
static def findHouseCountByDate(def city, def region, def name, def alias, def cat, def startDate){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
if(name || alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
static def findHousePriceByDate(def city, def region, def name, def alias, def cat, def startDate){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
if(name || alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
def prices = []
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
if(map.day_rent_price){
prices << map.day_rent_price
}
}
}
return prices
}
static def findNewPoiDatas(city){
String sql = "select cityname,adname,name,address,location from poi_amap_2024 where (cityname='烟台市' or cityname='北京市') and typecode=120201"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
index++
println index
rets << [map.cityname, map.adname, map.name, map.address, map.location.x, map.location.y]
})
return rets;
}
static def findAllNewPoiDatas(city){
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
// }
})
return rets;
}
static def findOneThirdNetWorkOffice(city, region, name){
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
def map = [:]
ElasticUtils.executeSqlQuery(sql, { m->
if(m.location && !map){
map = ["lng":m.location.x, "lat":m.location.y]
}
})
return map
}
static def findThirdNetWorkOffice(){
String sql = "select city,region,name,address,location from office where city='烟台' or city='北京'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
index++
println index
if(map.location){
rets << [map.city, map.region, map.name, map.address, map.location.lon, map.location.lat]
}else{
rets << [map.city, map.region, map.name, map.address, "", ""]
}
})
return rets;
}
/**
* 是否是别墅
* @return
@@ -4773,6 +4917,21 @@ class CommonDatautils {
return cityMap
}
static def findFullNameCitys(){
def q = new BasicDBObject()
q.put("type","城市")
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("province", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
def cityList = []
for(def d : dbcursor){
cityList << d
}
return cityList
}
static def queryDistNameByCity(){
def q = new BasicDBObject()
@@ -4991,6 +5150,34 @@ class CommonDatautils {
}
static def queryFullRegionTownByCity(city, fields){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
for(def key : fields){
keys.put(key,1)
}
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
if(d.district){
def list = townMap.get(d.district)
if(!list){
list = []
}
list << d
townMap.put(d.district, list)
}
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryTownByCity(city){
@@ -5259,4 +5446,73 @@ class CommonDatautils {
return STATS_INDEX_COLL.findOne(q)
}
static def findAllThirdPrice(city, region, name, statDate){
def query = new BasicDBObject()
query.append("category", "district")
query.append("city", city)
query.append("region", region)
query.append("name", name)
query.append("s_date", new BasicDBObject(QueryOperators.GTE, statDate))
def dbCursor = thirdpartColl.find(query).sort(new BasicDBObject("s_date",-1))
def retMap = [:]
for(def d : dbCursor){
if(!retMap.get("xingfuli") && d.get("xingfuli_esf_price")){
retMap.put("xingfuli", d.get("xingfuli_esf_price"))
}
if(!retMap.get("anjuke") && d.get("anjuke_esf_price")){
retMap.put("anjuke", d.get("anjuke_esf_price"))
}
if(!retMap.get("soufang") && d.get("soufang_esf_price")){
retMap.put("soufang", d.get("soufang_esf_price"))
}
if(!retMap.get("lianjia") && d.get("lianjia_esf_price")){
retMap.put("lianjia", d.get("lianjia_esf_price"))
}
if(!retMap.get("lianjia") && d.get("beike_esf_price")){
retMap.put("lianjia", d.get("beike_esf_price"))
}
if(!retMap.get("cityhouse") && d.get("cityhouse_esf_price")){
retMap.put("cityhouse", d.get("cityhouse_esf_price"))
}
if(!retMap.get("jiwu") && d.get("jiwu_esf_price")){//使用吉屋替换房探 待抓取
retMap.put("jiwu", d.get("jiwu_esf_price"))
}
if(!retMap.get("house365") && d.get("365house_esf_price")){//使用淘房365替换房多多
retMap.put("house365", d.get("365house_esf_price"))
}
if(!retMap.get("maitian") && d.get("maitian_esf_price")){
retMap.put("maitian", d.get("maitian_esf_price"))
}
if(!retMap.get("tc_58") && d.get("58tc_esf_price")){
retMap.put("tc_58", d.get("58tc_esf_price"))
}
}
return retMap;
}
static getSeaweedArroundByCat(city, lng, lat, cat) {
try {
def query = new BasicDBObject()
query.append("cat", cat)
query.append("city", city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
query.append('$sd', sdq)
return DbUtils.seaweed.find(query).toList()
} catch (Exception e) {
e.printStackTrace()
}
return 0
}
}
@@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
import com.rabbitmq.client.AMQP
class AddCityRegionPrice {
@@ -24,7 +22,7 @@ class AddCityRegionPrice {
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
task.addCityprice()
// task.addCityprice()
// task.addDistrictprice()
// task.addCityRentPrice()
// task.addRegionRentPrice()
@@ -32,7 +30,7 @@ class AddCityRegionPrice {
// task.updateRegionRentData()
// task.updateDealPriceAreaType()
// task.checkCityImport()
task.checkCityImport()
}
def addRegionPrice(){
@@ -255,7 +253,7 @@ class AddCityRegionPrice {
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410")
months.each {m->
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
if(!obj){
@@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
@@ -54,8 +57,8 @@ class AddDistrictShapeBatch extends BaseBatch{
//
// }
// checkRangeCount()
synch100()
checkRangeCount()
// synch100()
}
def baiduMatch(city,districts){
@@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{
def checkRangeCount(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","乡镇街道")).batchSize(100)
def count = 0
def index =0
for(def d : dbcursor){
index ++
if(d.location){
def coordinates = d.location.coordinates
if(d.shape){
def coordinates = d.shape.coordinates
if(coordinates.size()>1){
count++
println count +" "+ d.city +" "+d.name
println count +" "+ d.city +" "+d.street
}
}else{
println d.city + " "+d.name
println d.city + " "+d.street
}
if(index%1000==0){
@@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 小区自动入库(新增小区,每月运行一次)
*/
@@ -36,22 +35,39 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = ["上海"]//CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
// //查找第三方匹配没有匹配上的小区
// def citys = CommonDatautils.findAllCity()
// def list = []
// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
// for(def c : citys){
// def noMatchDatas = findNoMatchResult(c)
// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
// def rets = checkDatas(noMatchDatas.keySet())
// list.addAll(rets)
// }
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
fileName = "D:\\数据\\add_dist_info_1.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
// ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
}
def check(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中已存在"
}else{
dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中别名中存在"
}
}
})
}
def checkDatas(keysets){
@@ -72,11 +88,23 @@ class AutoAddDistrictBatch extends BaseBatch{
//检测有没有房源
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//检查百度地图有没有
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
def baidu_obj = findBaiduCount(arrs[0], arrs[1], arrs[2])
def baidu_count = 0
def baidu_type = null
if(baidu_obj){
baidu_count = 1
baidu_type = baidu_obj.std_tag
}
//检查高德地图有没有
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_count = 0
def gaode_type = null
if(gaode_obj){
gaode_count = 1
gaode_type = gaode_obj.type
}
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -123,18 +151,18 @@ class AutoAddDistrictBatch extends BaseBatch{
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
return 1
return obj
}
}
}
return 0
return null
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") && d.shape){
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
obj = d
}
})
@@ -228,7 +256,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def count = BAIDU_POI_COLL.count(q)
return count;
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
}
@@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.joda.time.Instant
import java.time.format.DateTimeFormatter
import java.util.regex.Pattern
class DevProcess {
@@ -64,6 +58,7 @@ class DevProcess {
// task.deleteAddr()
// task.deleteAddr2()
task.synchDevTo100()
// task.statShape()
// task.addNewDev2()
// task.addAlias()
@@ -87,6 +82,8 @@ class DevProcess {
// task.matchDevShape()//匹配开发区围栏
// task.addQiDevResult() //获取齐梁数据结果
// task.updateDevShape()//修改开发区结构
}
@@ -794,6 +791,18 @@ class DevProcess {
development_zone_CONN.insert(rets)
}
def statShape(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true)))
def index = 0
for(def r : dbcursor){
index ++
def coordinates = r.shape.coordinates
println index+" "+r.name+" "+coordinates.size()
}
}
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
@@ -1148,4 +1157,26 @@ class DevProcess {
}
},0)
}
def updateDevShape(){
def q = new BasicDBObject()
q.put("shape", new BasicDBObject('$exists', true))
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q)
for(def d : dbcursor){
if(d.name!="福州长乐国际机场综合保税区"){
def udata = new BasicDBObject()
if(d.shape.coordinates){
d.shape.coordinates = [d.shape.coordinates]
udata.put("shape", d.shape)
}
if(d.bd_shape && d.bd_shape.coordinates){
d.bd_shape.coordinates = [d.bd_shape.coordinates]
udata.put("bd_shape", d.bd_shape)
}
def updateQuery = new BasicDBObject("_id",d._id)
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
@@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -33,7 +32,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
def path = "D:\\数据\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
//
@@ -168,7 +167,7 @@ class DevUpdateTestBatch {
println d['A'] +" 不存在"
}
}
},2)
},0)
}
@@ -1,17 +1,28 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.SerializationUtils
import org.bson.types.ObjectId
/**
* 查找重复小区
*/
class DistrictRepeatBatch {
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictRepeatBatch()
task.run()
// task.run()
task.resultProcess()
}
def run(){
@@ -65,4 +76,118 @@ class DistrictRepeatBatch {
return rows;
}
def resultProcess(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d->
if(d["O"] && !d["O"].contains("不处理") && !d['R']){
if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){
def updteId = d['H']
def q = new BasicDBObject()
q.put("_id", new ObjectId(updteId))
def udata = new BasicDBObject()
if(d['O']=="K列改为商住" ){
udata.put("estate_type2", "商住")
}else{
udata.put("estate_type2", "别墅")
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "update")
d["R"] = "库中已处理"
}
else if(d["O"] == "都删"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata))
saveOptLog(d["B"], d["C"], d["D"], "remove")
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "remove")
d["R"] = "库中已处理"
}
else if(d["O"]=="删除K列小区"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "remove")
def obj = CommonDatautils.findDistrictById(d['A'])
if(d['C']==d['J']){
def alias = []
def addr_alias = []
if(d['D']!=d['K']){
def s1 = d['D'].replace(d['K'], "")
def s2 = d['K'].replace(d['D'], "")
def addNameAlias = true
if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){
addNameAlias = false
}
if(addNameAlias){
if(obj.alias){
alias = SerializationUtils.clone(obj.alias)
}
if(!alias.contains(d['K'])){
alias << d['K']
}
}
}
def address = null
if(d['M'] && d['F']!=d['M']){
def formatAddress = AddressUtil.formatAddress(d['A'], d['M'])
if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){
if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){
if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){
address = formatAddress
}else{
if(obj.addr_alias){
addr_alias = SerializationUtils.clone(obj.addr_alias)
}
if(!addr_alias.contains(formatAddress)){
addr_alias << formatAddress
}
}
}
}
}
if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){
def udata2 = new BasicDBObject()
if(alias){
udata2.put("alias", alias)
}
if(addr_alias){
udata2.put("addr_alias", addr_alias)
}
if(address){
udata2.put("address", address)
}
udata2.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2))
}
}
d["R"] = "库中已处理"
}
}
})
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", type)
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -0,0 +1,349 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 新版poi处理入库(第二批)
*/
@BatchOwner(Watcher.GaoJuan)
class NewGaoDePoiBatch extends BaseBatch{
public static def version = new Date().getTime()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static def TAGS = []
static main(args) {
runBatch(new NewGaoDePoiBatch(), args)
}
@Override
void process(Object args) {
run()
}
def run(){
if (CommonDatautils.isWindow()) {
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
}else{
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
}
def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){
// if(cityObj.city == "胡杨河市"){
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
println cityObj.province+" " +cityObj.city+" "+results.size()
if(results){
def rows = running(cityObj.city, results)
save(rows)
}
// }
}
}
def getAllPoiType(filePath){
def poiTypes = ExcelUtils.read(filePath, 0, 1)
def types = []
for(def poi : poiTypes){
types << poi['A']
}
return types
}
def running(city,results){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
def mzbRegionMap = regionFullNameMap.get(city)
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets.addAll(createData(map, mzbRegionMap, townMap))
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def getStandardRegion(mzbRegionMap, district){
def tjjObj = mzbRegionMap.get(district)
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name == district){
return district
}
def updateRegion = null
if(tjjObj){
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=district){
updateRegion = tjjObj.mzb_district_name
}
}else{
def set = mzbRegionMap.keySet()
for(def key : set){
def tempObj = mzbRegionMap.get(key)
def alias = tempObj.alias
if(alias.contains(district) || district==tempObj.mzb_district_name){
updateRegion = key
break;
}
}
}
return updateRegion
}
def createData(map, mzbRegionMap, townMap){
def saveObj = [:]
saveObj.province = map.province
saveObj.city = map.city
def newDistrict = getStandardRegion(mzbRegionMap, map.district)
if(newDistrict){
saveObj.district = newDistrict
}else{
println map.city +" "+map.district +" 格式化不出区域"
}
saveObj.name = map.name
if(map.cat){
saveObj = catWash(saveObj, map)
}
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
location.lat = DataHandleUtils.getDoubleRound(6, map.lat)
saveObj.location = location
saveObj.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
saveObj.version = version
saveObj.data_id = map.map_id
saveObj.data_source = "高德地图"
saveObj.data_source_year = "2024-11"
if(saveObj.cat && TAGS.contains(saveObj.cat)){
saveObj.status = 0
}else{
saveObj.status = 1
}
def towns = townMap.get(saveObj.district)
if(location){
saveObj = addTown(saveObj, towns, location)
}
if(map.address){
saveObj = addressWash(saveObj, map, mzbRegionMap, towns)
}
return saveObj
}
def catWash(saveObj, map){
def arrs = map.cat.split(";|:")
if(arrs.size()>=1){
saveObj.type = arrs[0]
}
if(arrs.size()>=2){
saveObj.secondary_type = arrs[1]
}else{
saveObj.secondary_type = saveObj.type
}
if(arrs.size()>=3){
def smallstr = arrs[2]
def smallArr = null
if(smallstr.contains("|")){
smallArr = smallstr.split("|")
}
if(smallArr && smallArr.size()>=1){
saveObj.small_type = smallArr[0]
}else{
saveObj.small_type = smallstr
}
}else{
saveObj.small_type = saveObj.secondary_type
}
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
return saveObj
}
def addTown(saveObj, towns, location){
if(towns){
for(def town : towns){
if(town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
saveObj.township = town.street
break
}
}
}
// if(map.address && !saveObj.township){
// for(def town : towns){
// if(map.address.contains(town.street)){
// saveObj.township = town.street
// break
// }
// }
// if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
// println map.address +" 含街道,镇,乡"
// }
// }
}
return saveObj
}
def addressWash(saveObj, map, mzbRegionMap, towns){
if(map.address.length()<3){
saveObj.address = ""
}else{
def address = map.address
def i = 0
while(i<5){
if(address && address.toString().startsWith(map.province)){
address = address.toString().substring(map.province.toString().length())
}
if(address && address.toString().startsWith(map.city)){
address = address.toString().substring(map.city.toString().length())
}
if(address && saveObj.district){
def tjjObj = mzbRegionMap.get(saveObj.district)
if(tjjObj){
def alias = null
if(tjjObj.alias){
alias = new CopyOnWriteArrayList<>(tjjObj.alias);
}else{
alias = new CopyOnWriteArrayList<>();
}
if(!alias.contains(saveObj.district)){
alias << saveObj.district
}
if(alias.size()>=2){
alias = alias.sort(){ a, b ->
b.length().compareTo(a.length());
}
}
for(def regionName : alias){
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
address = address.substring(regionName.toString().length())
break
}
}
}
}
if(address && address.toString().startsWith("市辖区")){
address = address.toString().substring(3)
}
if(address && towns){
for(def town : towns){
def townAlias = null
if(town.alias){
townAlias = new CopyOnWriteArrayList<>(town.alias);
}else{
townAlias = new CopyOnWriteArrayList<>();
}
if(!townAlias.contains(town.street)){
townAlias << town.street
}
if(townAlias.size()>=2){
townAlias = townAlias.sort(){ a, b ->
b.length().compareTo(a.length());
}
}
for(def townName : townAlias){
if(address.startsWith(townName)){
address = address.substring(townName.toString().length())
// if(!saveObj.township){
// saveObj.township = town.street
// }
break
}
}
}
}
i++
}
if(address && address.startsWith("(") && address.endsWith(")")){
saveObj.address = address.substring(1, address.length()-1)
}else{
saveObj.address = address
}
}
return saveObj
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 2000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
rows.each{map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
}
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class NewPoiData {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new NewPoiData()
task.run()
}
def run(){
// export()
// exportNetWorkOffice()
// matchByCity()
// noMatchData()
calcuteDistance()
}
def noMatchData(){
ExcelUtils.multiThreadingModify("d:\\数据\\未匹配写字楼2.xlsx", { d ->
if(d['H']!='地图'){
def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C'])
if(map){
d['E'] = map.lng
d['F'] = map.lat
}
}
})
}
def matchByCity(){
ExcelUtils.multiThreadingModify("d:\\数据\\入库写字楼.xlsx",{d->
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false)
def formatAddress = null
if(d['D']){
formatAddress = AddressUtil.formatAddress(d['A'], d['D'])
d['G'] = formatAddress
}
if(!jsonObject && formatAddress){
if(DataUtil.isPreciseAddress(formatAddress)){
def address = d['A']
if(d['B']){
address = address + d['B']
}
address = address + formatAddress
jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false)
}
}
if(jsonObject) {
d['H'] = jsonObject.get("id")
d['I'] = jsonObject.get("city")
d['J'] = jsonObject.get("district")
d['K'] = jsonObject.get("name")
d['L'] = jsonObject.get("propertyType")
d['M'] = jsonObject.get("credit")
}
})
}
def export(){
def rets = []
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
rets.addAll(CommonDatautils.findNewPoiDatas())
ExcelUtils.write("d:\\数据\\poi写字楼.xlsx", rets, "写字楼")
}
def exportNetWorkOffice(){
def rets = []
rets << ["城市", "区域", "名称", "地址", "经度", "纬度"]
rets.addAll(CommonDatautils.findThirdNetWorkOffice())
ExcelUtils.write("d:\\数据\\poi写字楼-网站.xlsx", rets, "写字楼")
}
def calcuteDistance(){
ExcelUtils.multiThreadingModify("d:\\数据\\写字楼测试-1118-新增.xlsx", { d ->
if(d['F']){
if(d['A'].contains("北京")){
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075)
d['K'] = distance
}else if(d['A'].contains("烟台")){
def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616)
d['K'] = distance
}
}
}, 1)
}
def addDistrict(){
def path = "D:\\数据\\入库写字楼.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def city = d['A']
def region = d['B']
def name = d['C']
def formatAddress = null
def address = d['D']
if(address){
if(d['G']){
formatAddress = AddressUtil.formatAddress(city, address)
}
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
def key = city+"##"+region+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","office")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
}
}
},0)
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
class PoiNineSplitBatch{
static main(args) {
def task = new PoiNineSplitBatch()
task.run()
}
// @Override
// void process(Object args) {
// run()
// }
def run(){
def i = 0;
while(i<360) {
def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\new_poi\\new_poi_address_split_${i}.xlsx"
}
File file = new File(fileName);
if (file.exists()) {
ExcelUtils.multiThreadingModify(fileName, { d ->
if (d['C'] && !d['G']) {
println d['C'] +" "+ d['E']
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E'])
if (jsonObject) {
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
d['G'] = jsonObject.get("street")
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
d['H'] = jsonObject.get("streetNumber")
}
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
d['I'] = jsonObject.get("township")
}
}
}
})
println fileName +" 九级拆分完成"
}else{
println fileName +" 文件不存在"
}
i++
}
}
}
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -41,12 +42,10 @@ class RegionStandardBatch extends BaseBatch{
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
def index = 0
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
for(def city : citys){
index ++
// if(index<32){
// continue
// }
if(runCitys.contains(city)){
continue
}
@@ -57,19 +56,32 @@ class RegionStandardBatch extends BaseBatch{
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
}
File file=new File(fileName);
if(file.exists()){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
// def rets = findEsPoiByCity(fullCityName)
def rets = readCSV(fileName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
println "没找区域:"+city
}
}else{
println "没找区域:"+city
}
}else{
println "没找区域:"+city
println fileName+" 文件没找到"
}
}else{
println "没找到相应的全称:"+city
}
@@ -365,6 +377,29 @@ class RegionStandardBatch extends BaseBatch{
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
try{
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}catch(Exception e){
println filename+" 文件异常"
}
}
}
return rets;
}
}
@@ -1,15 +1,14 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
class TempExportData {
@@ -17,7 +16,10 @@ class TempExportData {
static main(args) {
def task = new TempExportData()
task.run()
// task.run()
def rets = task.fzHouseStat()
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
println "文件已生成"
}
def run() {
@@ -75,4 +77,47 @@ class TempExportData {
println(city +" "+rets.size())
return rets;
}
def fzHouseStat(){
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "alias","estate_type2"])
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size()
def rets = new CopyOnWriteArrayList()
rets << ["城市","区域","名称","上月价格","3年房源量","2年房源量", "2年第三方家数"]
for(def i=0;i<districts.size();i++){
final def o = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def prices = CommonDatautils.queryDistrictPrices(o.city, o.region, o.name, "2024-10-01", "2024-10-31")
def avg_price = 0
if(prices){
avg_price = prices[0]
}
def count1 = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20211115)
def count2 = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20221115)
def thirdMap = CommonDatautils.findAllThirdPrice(o.city, o.region, o.name,202211)
def thirdCount = 0
if(thirdMap){
thirdCount = thirdMap.size()
}
rets << [o.city, o.region, o.name, avg_price, count1, count2, thirdCount]
println addCount.get() + " / "+ total
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
}
@@ -1,105 +1,58 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.math.RandomUtils
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
def city = "昌江"
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
def date = d['B']
def start = DateUtils.getFirstDayOfMonth(date)
def end = DateUtils.getLastDayOfMonth(date)
def weekends = DateUtils.getWeekendBetweenDates(start, end)
for(def weekend : weekends){
def avg_price = d['I']
def val = random(-200,200)
def new_price = (int)(avg_price*val/10000 + avg_price)
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
def o = new BasicDBObject()
o.put("avg_price", new_price)
o.put("c_date" , new Date())
o.put("city", city)
o.put("m_date" , new Date())
o.put("weekend", weekend)
o.put("cat","城市")
o.put("moving_average" , (int)avg_price)
if(!last_price){
o.put("weekend_markup" , 0)
}else{
def weekend_markup = (int)((new_price-last_price)*100/last_price)
o.put("weekend_markup" , weekend_markup)
}
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
},1)
public static final int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
@@ -1,6 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -10,17 +10,15 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
static DBCollection data_process_coll = CrawalDBUtil.getDataProcessCol()
static main(args) {
runBatch(new ThardParthAnalyBatch(), args)
@@ -119,11 +117,14 @@ class ThardParthAnalyBatch extends BaseBatch{
}
def findMatchSuccessDistCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
def dbcursor = thirdPartyColl.find(q)
// def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
// def dbcursor = thirdPartyColl.find(q)
def q = new BasicDBObject().append("s_date", sDate)
def dbcursor = data_process_coll.find(q)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
set << d.match_city+"##"+d.match_region+"##"+d.match_name
}
return set.size()
}
@@ -4,7 +4,6 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
@@ -12,14 +11,11 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 第三方小区与我们小区映射关系表
*/
@@ -45,7 +41,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
// //接口进行匹配
//// //½Ó¿Ú½øÐÐÆ¥Åä
match()
addOldMatchResult()
@@ -53,7 +49,12 @@ class ThirdPriceMappingBatch extends BaseBatch{
def match(){
def citys = CommonUtils.findAllCity()
def index = 0
for(def c : citys){
index ++
if(index < 206){
continue
}
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
@@ -181,6 +182,10 @@ class ThirdPriceMappingBatch extends BaseBatch{
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
keys.put("source", 1)
for(def c : citys){
def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList()
@@ -200,8 +205,13 @@ class ThirdPriceMappingBatch extends BaseBatch{
void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString())
def obj = dataProcessColl.findOne(query)
//.append("s_date", curDate)
def query = new BasicDBObject().append("city", city).append("region",d.district).append("name", d.name).append("source",d.source)
def objs = dataProcessColl.find(query).sort(new BasicDBObject("s_date",-1)).limit(1)
def obj = null
if(objs){
obj = objs[0]
}
if(obj) {
def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district")
def udata = new BasicDBObject()
@@ -24,6 +24,7 @@ class TownStandardBatch extends BaseBatch{
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
def updateList = []
def addAliasList = []
static main(args) {
runBatch(new TownStandardBatch(), args)
@@ -66,7 +67,10 @@ class TownStandardBatch extends BaseBatch{
def townStandardByPoi(){
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
def citys = CommonDatautils.findAllCity()
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
//浙江,江苏,安徽
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -78,7 +82,7 @@ class TownStandardBatch extends BaseBatch{
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\${fullCityName}.csv"
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
}
File file=new File(fileName);
if(file.exists()){
@@ -88,7 +92,7 @@ class TownStandardBatch extends BaseBatch{
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
def set = rMap.keySet()
// def rets = findEsPoiByCity(fullCityName)
def rets = readPoiCsv(fullCityName)
def rets = readPoiCsv(fileName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
@@ -96,12 +100,16 @@ class TownStandardBatch extends BaseBatch{
// haveStreetProcess(city, set, cat)
}
}
}else{
println fileName+" 文件没找到"
}
}else{
println "没找到相应的全称:"+city
}
}
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
@@ -194,6 +202,18 @@ class TownStandardBatch extends BaseBatch{
}
break
}
// else{
// if(town.street.contains(d.township.replace("镇","").replace("街道","").replace("乡",""))){
// updateTown = town.street
// opt = false
// def addObj = [town.city, town.district, town.street, d.township]
// if(!addAliasList.contains(addObj)){
// addAliasList << addObj
// }
//
// break
// }
// }
}
}
if(opt){
@@ -206,11 +226,11 @@ class TownStandardBatch extends BaseBatch{
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
results << m
// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
@@ -256,7 +276,7 @@ class TownStandardBatch extends BaseBatch{
println("批量更新中有失败的操作:")
bulkResponse.buildFailureMessage().each { println(it) }
} else {
println("批量更新成功!")
// println("批量更新成功!")
}
} catch (Exception e) {
e.printStackTrace()
@@ -0,0 +1,115 @@
package com.fangjia.script.gj.xiamen
import org.bouncycastle.crypto.engines.SM4Engine;
import org.bouncycastle.crypto.modes.CBCBlockCipher;
import org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher;
import org.bouncycastle.crypto.params.KeyParameter;
import org.bouncycastle.crypto.params.ParametersWithIV;
import org.bouncycastle.util.encoders.Hex;
import java.security.SecureRandom;
/**
* @author shenjian
* @since 2024/10/9
*/
public class CryptoSm4Util {
/**
* 默认密钥
*/
private static final String DEFAULT_SECRET_KEY = "A1B2C3D4E5F60789A1B2C3D4E5F60789";
private static final String CHARSET = "UTF-8";
// String plaintext -> Base64-encoded String ciphertext
public static String encrypt(String key, String plaintext) {
try {
byte[] keyBytes = Hex.decode(key);
byte[] iv = generateIV();
PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
cipher.init(true, new ParametersWithIV(new KeyParameter(keyBytes), iv));
byte[] input = plaintext.getBytes(CHARSET);
byte[] encrypted = new byte[cipher.getOutputSize(input.length)];
int length1 = cipher.processBytes(input, 0, input.length, encrypted, 0);
int length2 = cipher.doFinal(encrypted, length1);
byte[] finalEncrypted = new byte[length1 + length2];
System.arraycopy(encrypted, 0, finalEncrypted, 0, finalEncrypted.length);
return Hex.toHexString(iv) + Hex.toHexString(finalEncrypted);
} catch (Exception ex) {
ex.printStackTrace();
}
return null;
}
private static byte[] generateIV() {
byte[] iv = new byte[16];
new SecureRandom().nextBytes(iv);
return iv;
}
// Base64-encoded String ciphertext -> String plaintext
public static String decrypt(String key, String ciphertext) {
try {
byte[] keyBytes = Hex.decode(key);
byte[] iv = Hex.decode(ciphertext.substring(0, 32));
byte[] encryptedData = Hex.decode(ciphertext.substring(32));
PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
cipher.init(false, new ParametersWithIV(new KeyParameter(keyBytes), iv));
byte[] decrypted = new byte[cipher.getOutputSize(encryptedData.length)];
int length1 = cipher.processBytes(encryptedData, 0, encryptedData.length, decrypted, 0);
int length2 = cipher.doFinal(decrypted, length1);
byte[] finalDecrypted = new byte[length1 + length2];
System.arraycopy(decrypted, 0, finalDecrypted, 0, finalDecrypted.length);
return new String(finalDecrypted, CHARSET).trim();
} catch (Exception ex) {
ex.printStackTrace();
}
return null;
}
/**
* 使用默认密钥加密
*
* @param str 加密内容
* @return 以String密文输入, String明文输出
*/
public static String encode(String str) {
return encode(DEFAULT_SECRET_KEY, str);
}
/**
* 自定义密钥加密
*
* @param key 自定义密钥
* @param str 加密的内容
* @return String明文输入, String密文输出
*/
public static String encode(String key, String str) {
return encrypt(key, str);
}
/**
* 使用默认密钥解密
*
* @param str 解密内容
* @return 以String密文输入, String明文输出
*/
public static String decode(String str) {
return decode(DEFAULT_SECRET_KEY, str);
}
/**
* 自定义密钥解密
*
* @param key 自定义密钥
* @param str 加密内容
* @return 以String密文输入, String明文输出
*/
public static String decode(String key, String str) {
return decrypt(key, str);
}
public static void main(def a){
println encrypt( DEFAULT_SECRET_KEY, "测试")
}
}
@@ -15,10 +15,14 @@ import com.mongodb.DBCollection
/**
* 厦门涉台园区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class RelatedTaiWanBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
@@ -28,6 +32,8 @@ class RelatedTaiWanBatch extends BaseBatch{
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
static main(args) {
runBatch(new RelatedTaiWanBatch(), args);
}
@@ -37,14 +43,11 @@ class RelatedTaiWanBatch extends BaseBatch{
def day = new Date().getDate()
if(day == 10){
//createDatas() //企业库获取涉台园区企业
// createDatas() //企业库获取涉台园区企业
updateRangeByPoint() //更改企业所在自贸区
export() //导出数据
}else{
exportEmptyFile()
}
}
@@ -152,10 +155,18 @@ class RelatedTaiWanBatch extends BaseBatch{
}
def allDatas = []
allDatas.addAll(rows)
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
//原不加密之前
// FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
// XmBankFtpUtils.ftpUpload(fileName, curDate)
def str1 = FileUtils.createFileStringByCustomSuffix(rows, fileName, "utf-8", "|@|","|+|");
def str1_encryptedText = CryptoSm4Util.encrypt( CryptoSm4Util.DEFAULT_SECRET_KEY, str1)
FileUtils.createEncryptFile(fileName, str1_encryptedText, false)
XmBankFtpUtils.ftpUpload(fileName, curDate)
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
if(CommonDatautils.isWindow()){
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
@@ -181,4 +192,7 @@ class RelatedTaiWanBatch extends BaseBatch{
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
XmBankFtpUtils.ftpUpload(fileName2, curDate)
}
}
@@ -0,0 +1,134 @@
package com.fangjia.script.gj.zhongshu
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
class DistrictArrountOffice {
static main(args) {
def task = new DistrictArrountOffice()
task.run()
}
def run(){
def dataMap = ["北京":"116.481075,39.996743", "烟台":"121.438616,37.459645"]
def rets = []
rets << ["城市","区域","名称","地址","经度","纬度","写字楼等级","租金(起)","租金(止)", "价格来源"]
dataMap.each {k,v->
def locations = v.split(",")
def arroundOffices = CommonDatautils.getSeaweedArroundByCat(k, locations[0], locations[1], "office")
println k +" "+arroundOffices.size()
if(arroundOffices){
def index = 0
for(def o : arroundOffices){
index++
def priceMap = findEsOffice(o.city, o.region, o.name)
if(!priceMap){
priceMap = findEsHouse(o, "20231112")
}
def start_price = null
def end_price = null
def source = null
if(priceMap){
if(priceMap.avg_start_price){
start_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_start_price)
}
if(priceMap.avg_end_price){
end_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_end_price)
}
source = priceMap.source
}
def lng = DataHandleUtils.getDoubleRound(6, o.lng/10000000000.0)
def lat = DataHandleUtils.getDoubleRound(6, o.lat/10000000000.0)
def row = [o.city, o.region, o.name, o.address, lng, lat, o.bed_kind, start_price, end_price, source]
rets << row
println index +" "+ row
}
}
}
ExcelUtils.write("d:\\数据\\写字楼测试-1118.xlsx", rets, "数据")
}
def findEsOffice(city, region, name){
def map = [:]
def start_rent_prices = []
def end_rent_prices = []
def rent_prices = []
ElasticUtils.executeSqlQuery("select * from office where city='${city}' and region='${region}' and name='${name}'", { r->
if(r.start_rent_price){
start_rent_prices << r.start_rent_price
}
if(r.end_rent_price){
end_rent_prices << r.end_rent_price
}
if(r.rent_price){
rent_prices << r.rent_price
}
})
def avg_start_price = null
if(start_rent_prices){
avg_start_price = start_rent_prices.sum()/start_rent_prices.size()
}
def avg_end_price = null
if(end_rent_prices){
avg_end_price = end_rent_prices.sum()/end_rent_prices.size()
}
if(!avg_start_price && !avg_end_price){
if(rent_prices){
if(rent_prices.size()>1){
rent_prices = rent_prices.sort()
avg_start_price = rent_prices[0]
avg_end_price = rent_prices[rent_prices.size()-1]
}else{
avg_start_price = rent_prices[0] - rent_prices[0]*0.2
avg_end_price = rent_prices[0] + rent_prices[0]*0.2
}
}
}
if(avg_start_price){
map.put("avg_start_price", avg_start_price)
}
if(avg_end_price){
map.put("avg_end_price", avg_end_price)
}
if(map){
map.source = "非房源"
}
return map
}
def findEsHouse(o, startDate){
def rent_prices = CommonDatautils.findHousePriceByDate(o.city, o.region, o.name, o.alias, "office_rent", startDate)
def avg_start_price = null
def avg_end_price = null
def map = [:]
if(rent_prices){
if(rent_prices.size()>1){
rent_prices = rent_prices.sort()
avg_start_price = rent_prices[0]
avg_end_price = rent_prices[rent_prices.size()-1]
}else{
avg_start_price = rent_prices[0] - rent_prices[0]*0.2
avg_end_price = rent_prices[0] + rent_prices[0]*0.2
}
}
if(avg_start_price){
map.put("avg_start_price", avg_start_price)
}
if(avg_end_price){
map.put("avg_end_price", avg_end_price)
}
if(map){
map.source = "房源"
}
return map
}
}
@@ -7,21 +7,8 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.CommonUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DB
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import com.fangjia.script.util.*
import com.mongodb.*
import org.apache.commons.lang3.SerializationUtils
import org.apache.commons.lang3.StringUtils
import org.apache.tools.ant.util.DateUtils
@@ -31,9 +18,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 第三方数据标准化处理
* 成功type说明
@@ -689,14 +674,15 @@ class DataStandardBatch extends BaseBatch{
def pre = (data.avg_price-price)*100/price
if(pre>=-20 && pre<=20){
return dbo;
}else{
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","·¿Ô´¼Û¸ñ¹ýÂË")
THIRD_PRICE_FILTER_COLL.save(data)
}
return null
}
// else{
// def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
// if(!ret){
// data.put("msg","房源价格过滤")
// THIRD_PRICE_FILTER_COLL.save(data)
// }
// return null
// }
}
}
return dbo
@@ -934,7 +920,8 @@ class DataStandardBatch extends BaseBatch{
DBObject qdata = new BasicDBObject();
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","上海")
// qdata.put("name","°×½ð嫹¬")
// qdata.put("name","绿地云天坊")
// qdata.put("status", 3)
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));