Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-11-20 16:46:45 +08:00
13 changed files with 625 additions and 60 deletions
+7
View File
@@ -530,6 +530,13 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15to18</artifactId>
<version>1.70</version> <!-- 确保使用最新版本 -->
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -2656,12 +2656,12 @@ class CommonDatautils {
}
public static def queryDistrictPrices(city, region, name, estateType, startDate, endDate){
public static def queryDistrictPrices(city, region, name, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("estate_type", estateType);
// queryCondition.put("estate_type", estateType);
queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate)));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
def prices = [];
@@ -4107,6 +4107,136 @@ class CommonDatautils {
}
static def findHouseCountByDate(def city, def region, def name, def alias, def cat, def startDate){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
if(name || alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
static def findHousePriceByDate(def city, def region, def name, def alias, def cat, def startDate){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
if(name || alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
def prices = []
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
if(map.day_rent_price){
prices << map.day_rent_price
}
}
}
return prices
}
static def findNewPoiDatas(city){
String sql = "select cityname,adname,name,address,location from poi_amap_2024 where (cityname='烟台市' or cityname='北京市') and typecode=120201"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
index++
println index
rets << [map.cityname, map.adname, map.name, map.address, map.location.x, map.location.y]
})
return rets;
}
static def findOneThirdNetWorkOffice(city, region, name){
String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
def map = [:]
ElasticUtils.executeSqlQuery(sql, { m->
if(m.location && !map){
map = ["lng":m.location.x, "lat":m.location.y]
}
})
return map
}
static def findThirdNetWorkOffice(){
String sql = "select city,region,name,address,location from office where city='烟台' or city='北京'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
index++
println index
if(map.location){
rets << [map.city, map.region, map.name, map.address, map.location.lon, map.location.lat]
}else{
rets << [map.city, map.region, map.name, map.address, "", ""]
}
})
return rets;
}
/**
* 是否是别墅
* @return
@@ -5259,4 +5389,73 @@ class CommonDatautils {
return STATS_INDEX_COLL.findOne(q)
}
static def findAllThirdPrice(city, region, name, statDate){
def query = new BasicDBObject()
query.append("category", "district")
query.append("city", city)
query.append("region", region)
query.append("name", name)
query.append("s_date", new BasicDBObject(QueryOperators.GTE, statDate))
def dbCursor = thirdpartColl.find(query).sort(new BasicDBObject("s_date",-1))
def retMap = [:]
for(def d : dbCursor){
if(!retMap.get("xingfuli") && d.get("xingfuli_esf_price")){
retMap.put("xingfuli", d.get("xingfuli_esf_price"))
}
if(!retMap.get("anjuke") && d.get("anjuke_esf_price")){
retMap.put("anjuke", d.get("anjuke_esf_price"))
}
if(!retMap.get("soufang") && d.get("soufang_esf_price")){
retMap.put("soufang", d.get("soufang_esf_price"))
}
if(!retMap.get("lianjia") && d.get("lianjia_esf_price")){
retMap.put("lianjia", d.get("lianjia_esf_price"))
}
if(!retMap.get("lianjia") && d.get("beike_esf_price")){
retMap.put("lianjia", d.get("beike_esf_price"))
}
if(!retMap.get("cityhouse") && d.get("cityhouse_esf_price")){
retMap.put("cityhouse", d.get("cityhouse_esf_price"))
}
if(!retMap.get("jiwu") && d.get("jiwu_esf_price")){//使用吉屋替换房探 待抓取
retMap.put("jiwu", d.get("jiwu_esf_price"))
}
if(!retMap.get("house365") && d.get("365house_esf_price")){//使用淘房365替换房多多
retMap.put("house365", d.get("365house_esf_price"))
}
if(!retMap.get("maitian") && d.get("maitian_esf_price")){
retMap.put("maitian", d.get("maitian_esf_price"))
}
if(!retMap.get("tc_58") && d.get("58tc_esf_price")){
retMap.put("tc_58", d.get("58tc_esf_price"))
}
}
return retMap;
}
static getSeaweedArroundByCat(city, lng, lat, cat) {
try {
def query = new BasicDBObject()
query.append("cat", cat)
query.append("city", city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
query.append('$sd', sdq)
return DbUtils.seaweed.find(query).toList()
} catch (Exception e) {
e.printStackTrace()
}
return 0
}
}
@@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
import com.rabbitmq.client.AMQP
class AddCityRegionPrice {
@@ -24,7 +22,7 @@ class AddCityRegionPrice {
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
task.addCityprice()
// task.addCityprice()
// task.addDistrictprice()
// task.addCityRentPrice()
// task.addRegionRentPrice()
@@ -32,7 +30,7 @@ class AddCityRegionPrice {
// task.updateRegionRentData()
// task.updateDealPriceAreaType()
// task.checkCityImport()
task.checkCityImport()
}
def addRegionPrice(){
@@ -255,7 +253,7 @@ class AddCityRegionPrice {
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410")
months.each {m->
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
if(!obj){
@@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 小区自动入库(新增小区,每月运行一次)
*/
@@ -37,7 +36,7 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = ["上海"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
@@ -52,6 +51,18 @@ class AutoAddDistrictBatch extends BaseBatch{
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
}
def check(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中已存在"
}
})
}
def checkDatas(keysets){
@@ -72,11 +83,23 @@ class AutoAddDistrictBatch extends BaseBatch{
//检测有没有房源
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//检查百度地图有没有
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
def baidu_obj = findBaiduCount(arrs[0], arrs[1], arrs[2])
def baidu_count = 0
def baidu_type = null
if(baidu_obj){
baidu_count = 1
baidu_type = baidu_obj.std_tag
}
//检查高德地图有没有
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_count = 0
def gaode_type = null
if(gaode_obj){
gaode_count = 1
gaode_type = gaode_obj.type
}
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -123,18 +146,18 @@ class AutoAddDistrictBatch extends BaseBatch{
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
return 1
return obj
}
}
}
return 0
return null
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") && d.shape){
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
obj = d
}
})
@@ -228,7 +251,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def count = BAIDU_POI_COLL.count(q)
return count;
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
}
@@ -1,17 +1,28 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.SerializationUtils
import org.bson.types.ObjectId
/**
* 查找重复小区
*/
class DistrictRepeatBatch {
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictRepeatBatch()
task.run()
// task.run()
task.resultProcess()
}
def run(){
@@ -65,4 +76,118 @@ class DistrictRepeatBatch {
return rows;
}
def resultProcess(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区重复数据处理.xlsx",{d->
if(d["O"] && !d["O"].contains("不处理") && !d['R']){
if(d['O']=="K列改为商住" || d["O"] == "修改K列为别墅"){
def updteId = d['H']
def q = new BasicDBObject()
q.put("_id", new ObjectId(updteId))
def udata = new BasicDBObject()
if(d['O']=="K列改为商住" ){
udata.put("estate_type2", "商住")
}else{
udata.put("estate_type2", "别墅")
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "update")
d["R"] = "库中已处理"
}
else if(d["O"] == "都删"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata))
saveOptLog(d["B"], d["C"], d["D"], "remove")
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "remove")
d["R"] = "库中已处理"
}
else if(d["O"]=="删除K列小区"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
saveOptLog(d["I"], d["J"], d["K"], "remove")
def obj = CommonDatautils.findDistrictById(d['A'])
if(d['C']==d['J']){
def alias = []
def addr_alias = []
if(d['D']!=d['K']){
def s1 = d['D'].replace(d['K'], "")
def s2 = d['K'].replace(d['D'], "")
def addNameAlias = true
if(s1=="小区" || s2=="小区" || s1=="别墅" || s2 == "别墅" || s1=="商住" || s2=="商住"){
addNameAlias = false
}
if(addNameAlias){
if(obj.alias){
alias = SerializationUtils.clone(obj.alias)
}
if(!alias.contains(d['K'])){
alias << d['K']
}
}
}
def address = null
if(d['M'] && d['F']!=d['M']){
def formatAddress = AddressUtil.formatAddress(d['A'], d['M'])
if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){
if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){
if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){
address = formatAddress
}else{
if(obj.addr_alias){
addr_alias = SerializationUtils.clone(obj.addr_alias)
}
if(!addr_alias.contains(formatAddress)){
addr_alias << formatAddress
}
}
}
}
}
if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){
def udata2 = new BasicDBObject()
if(alias){
udata2.put("alias", alias)
}
if(addr_alias){
udata2.put("addr_alias", addr_alias)
}
if(address){
udata2.put("address", address)
}
udata2.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2))
}
}
d["R"] = "库中已处理"
}
}
})
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", type)
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -41,12 +42,11 @@ class RegionStandardBatch extends BaseBatch{
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
def index = 0
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
for(def city : citys){
index ++
// if(index<32){
// continue
// }
if(runCitys.contains(city)){
continue
}
@@ -57,19 +57,32 @@ class RegionStandardBatch extends BaseBatch{
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
}
File file=new File(fileName);
if(file.exists()){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
// def rets = findEsPoiByCity(fullCityName)
def rets = readCSV(fileName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
println "没找区域:"+city
}
}else{
println "没找区域:"+city
}
}else{
println "没找区域:"+city
println fileName+" 文件没找到"
}
}else{
println "没找到相应的全称:"+city
}
@@ -365,6 +378,29 @@ class RegionStandardBatch extends BaseBatch{
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
try{
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}catch(Exception e){
println filename+" 文件异常"
}
}
}
return rets;
}
}
@@ -1,15 +1,14 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
class TempExportData {
@@ -17,7 +16,10 @@ class TempExportData {
static main(args) {
def task = new TempExportData()
task.run()
// task.run()
def rets = task.fzHouseStat()
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
println "文件已生成"
}
def run() {
@@ -75,4 +77,47 @@ class TempExportData {
println(city +" "+rets.size())
return rets;
}
def fzHouseStat(){
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "alias","estate_type2"])
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size()
def rets = new CopyOnWriteArrayList()
rets << ["城市","区域","名称","上月价格","3年房源量","2年房源量", "2年第三方家数"]
for(def i=0;i<districts.size();i++){
final def o = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def prices = CommonDatautils.queryDistrictPrices(o.city, o.region, o.name, "2024-10-01", "2024-10-31")
def avg_price = 0
if(prices){
avg_price = prices[0]
}
def count1 = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20211115)
def count2 = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20221115)
def thirdMap = CommonDatautils.findAllThirdPrice(o.city, o.region, o.name,202211)
def thirdCount = 0
if(thirdMap){
thirdCount = thirdMap.size()
}
rets << [o.city, o.region, o.name, avg_price, count1, count2, thirdCount]
println addCount.get() + " / "+ total
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
}
@@ -4,7 +4,6 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
@@ -12,14 +11,11 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 第三方小区与我们小区映射关系表
*/
@@ -44,7 +40,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
@Override
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
addNemCommunity()
// //接口进行匹配
match()
@@ -66,7 +66,7 @@ class TownStandardBatch extends BaseBatch{
def townStandardByPoi(){
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
def citys = CommonDatautils.findAllCity()
def citys = ["福州"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -88,7 +88,7 @@ class TownStandardBatch extends BaseBatch{
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
def set = rMap.keySet()
// def rets = findEsPoiByCity(fullCityName)
def rets = readPoiCsv(fullCityName)
def rets = readPoiCsv(fileName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
@@ -96,6 +96,8 @@ class TownStandardBatch extends BaseBatch{
// haveStreetProcess(city, set, cat)
}
}
}else{
println fileName+" 文件没找到"
}
}else{
println "没找到相应的全称:"+city
@@ -201,16 +203,16 @@ class TownStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
results << m
// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
@@ -224,10 +226,10 @@ class TownStandardBatch extends BaseBatch{
}
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
if(results.size()>0){
updateByEs(results)
results.clear()
}
println fullCityName +" 处理完成"
}
@@ -0,0 +1,115 @@
package com.fangjia.script.gj.xiamen
import org.bouncycastle.crypto.engines.SM4Engine;
import org.bouncycastle.crypto.modes.CBCBlockCipher;
import org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher;
import org.bouncycastle.crypto.params.KeyParameter;
import org.bouncycastle.crypto.params.ParametersWithIV;
import org.bouncycastle.util.encoders.Hex;
import java.security.SecureRandom;
/**
* @author shenjian
* @since 2024/10/9
*/
public class CryptoSm4Util {
/**
* 默认密钥
*/
private static final String DEFAULT_SECRET_KEY = "A1B2C3D4E5F60789A1B2C3D4E5F60789";
private static final String CHARSET = "UTF-8";
// String plaintext -> Base64-encoded String ciphertext
public static String encrypt(String key, String plaintext) {
try {
byte[] keyBytes = Hex.decode(key);
byte[] iv = generateIV();
PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
cipher.init(true, new ParametersWithIV(new KeyParameter(keyBytes), iv));
byte[] input = plaintext.getBytes(CHARSET);
byte[] encrypted = new byte[cipher.getOutputSize(input.length)];
int length1 = cipher.processBytes(input, 0, input.length, encrypted, 0);
int length2 = cipher.doFinal(encrypted, length1);
byte[] finalEncrypted = new byte[length1 + length2];
System.arraycopy(encrypted, 0, finalEncrypted, 0, finalEncrypted.length);
return Hex.toHexString(iv) + Hex.toHexString(finalEncrypted);
} catch (Exception ex) {
ex.printStackTrace();
}
return null;
}
private static byte[] generateIV() {
byte[] iv = new byte[16];
new SecureRandom().nextBytes(iv);
return iv;
}
// Base64-encoded String ciphertext -> String plaintext
public static String decrypt(String key, String ciphertext) {
try {
byte[] keyBytes = Hex.decode(key);
byte[] iv = Hex.decode(ciphertext.substring(0, 32));
byte[] encryptedData = Hex.decode(ciphertext.substring(32));
PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
cipher.init(false, new ParametersWithIV(new KeyParameter(keyBytes), iv));
byte[] decrypted = new byte[cipher.getOutputSize(encryptedData.length)];
int length1 = cipher.processBytes(encryptedData, 0, encryptedData.length, decrypted, 0);
int length2 = cipher.doFinal(decrypted, length1);
byte[] finalDecrypted = new byte[length1 + length2];
System.arraycopy(decrypted, 0, finalDecrypted, 0, finalDecrypted.length);
return new String(finalDecrypted, CHARSET).trim();
} catch (Exception ex) {
ex.printStackTrace();
}
return null;
}
/**
* 使用默认密钥加密
*
* @param str 加密内容
* @return 以String密文输入, String明文输出
*/
public static String encode(String str) {
return encode(DEFAULT_SECRET_KEY, str);
}
/**
* 自定义密钥加密
*
* @param key 自定义密钥
* @param str 加密的内容
* @return String明文输入, String密文输出
*/
public static String encode(String key, String str) {
return encrypt(key, str);
}
/**
* 使用默认密钥解密
*
* @param str 解密内容
* @return 以String密文输入, String明文输出
*/
public static String decode(String str) {
return decode(DEFAULT_SECRET_KEY, str);
}
/**
* 自定义密钥解密
*
* @param key 自定义密钥
* @param str 加密内容
* @return 以String密文输入, String明文输出
*/
public static String decode(String key, String str) {
return decrypt(key, str);
}
public static void main(def a){
println encrypt( DEFAULT_SECRET_KEY, "测试")
}
}
@@ -15,10 +15,14 @@ import com.mongodb.DBCollection
/**
* 厦门涉台园区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class RelatedTaiWanBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
@@ -28,6 +32,8 @@ class RelatedTaiWanBatch extends BaseBatch{
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
static main(args) {
runBatch(new RelatedTaiWanBatch(), args);
}
@@ -37,14 +43,11 @@ class RelatedTaiWanBatch extends BaseBatch{
def day = new Date().getDate()
if(day == 10){
//createDatas() //企业库获取涉台园区企业
// createDatas() //企业库获取涉台园区企业
updateRangeByPoint() //更改企业所在自贸区
export() //导出数据
}else{
exportEmptyFile()
}
}
@@ -152,10 +155,18 @@ class RelatedTaiWanBatch extends BaseBatch{
}
def allDatas = []
allDatas.addAll(rows)
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
//原不加密之前
// FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
// XmBankFtpUtils.ftpUpload(fileName, curDate)
def str1 = FileUtils.createFileStringByCustomSuffix(rows, fileName, "utf-8", "|@|","|+|");
def str1_encryptedText = CryptoSm4Util.encrypt( CryptoSm4Util.DEFAULT_SECRET_KEY, str1)
FileUtils.createEncryptFile(fileName, str1_encryptedText, false)
XmBankFtpUtils.ftpUpload(fileName, curDate)
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
if(CommonDatautils.isWindow()){
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
@@ -181,4 +192,7 @@ class RelatedTaiWanBatch extends BaseBatch{
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
XmBankFtpUtils.ftpUpload(fileName2, curDate)
}
}
@@ -31,6 +31,7 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.client.solrj.util.ClientUtils;
import java.math.BigDecimal;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -1170,7 +1171,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService {
Integer minPrice = null;
Integer maxPrice = null;
if (districtStat != null && districtStat.getAvgPrice() > 0) {
int avgPrice = districtStat.getAvgPrice();
//int avgPrice = districtStat.getAvgPrice();
int avgPrice = new BigDecimal(districtStat.getAvgPrice()).multiply(new BigDecimal(0.9)).intValue();
minPrice = (int) (avgPrice * 0.8);
maxPrice = (int) (avgPrice * 1.2);
houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, minPrice));//0.85
@@ -2297,6 +2297,9 @@ public class DistrictServiceImpl implements DistrictService {
@SuppressWarnings("unchecked")
public List<District> findNearestDistrict(District district, int radiusMeters, int limit) {
if (district == null) {
return Collections.emptyList();
}
String cacheKey = district.getCity() + "#" + district.getRegion() + "#" + district.getName() + "#" + radiusMeters + "#" + limit;
Object obj = CacheUtils.getValue(this.getClass(), cacheKey);
if (obj != null) {