Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -183,17 +183,6 @@
|
||||
<artifactId>easyexcel</artifactId>
|
||||
<version>4.0.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.kafka</groupId>
|
||||
<artifactId>spring-kafka</artifactId>
|
||||
<version>2.8.9</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.18.28</version> <!-- 确保版本是最新的 -->
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<repositories>
|
||||
<repository>
|
||||
|
||||
-15
@@ -5,7 +5,6 @@ import com.fangjia.bkoff.model.DistrictModel;
|
||||
import com.fangjia.bkoff.model.JsonModel;
|
||||
import com.fangjia.bkoff.util.AddressParser;
|
||||
import com.fangjia.bkoff.util.GeocodeUtils;
|
||||
import com.fangjia.bkoff.util.KafkaUtils;
|
||||
import com.fangjia.core.Constants;
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.*;
|
||||
@@ -90,8 +89,6 @@ public class DistrictController {
|
||||
private DealPriceService dealPriceService;
|
||||
private AverageStatisticsService averageStatisticsService;
|
||||
|
||||
private KafkaUtils kafkaUtils;
|
||||
|
||||
|
||||
private static List<String> distTypes = Arrays.asList("多层", "小高层", "超高层", "高层", "花园洋房", "老洋房", "四合院");
|
||||
private static List<String> villaTypes = Arrays.asList("独栋", "双拼", "联排", "叠加");
|
||||
@@ -2288,14 +2285,6 @@ public class DistrictController {
|
||||
data.put("markMap", markMap);
|
||||
data.put("modificationDate", new Date());
|
||||
districtService.updateById(data, district.getId());
|
||||
JSONObject massage = new JSONObject();
|
||||
massage.put("city",district.getCity());
|
||||
massage.put("region",district.getRegion());
|
||||
massage.put("name",district.getName());
|
||||
massage.put("estateType2",district.getEstateType2());
|
||||
massage.put("id",district.getId());
|
||||
massage.put("type",2);
|
||||
kafkaUtils.sendMessage("seaweed-data-topic", massage.toString());
|
||||
//创建更新任务,用为更新常州农商行数据
|
||||
createJnTask(district.getId(), null);
|
||||
msg = "保存成功了!!!";
|
||||
@@ -4222,8 +4211,4 @@ public class DistrictController {
|
||||
public void setAverageStatisticsService(AverageStatisticsService averageStatisticsService) {
|
||||
this.averageStatisticsService = averageStatisticsService;
|
||||
}
|
||||
|
||||
public void setKafkaUtils(KafkaUtils kafkaUtils) {
|
||||
this.kafkaUtils = kafkaUtils;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
package com.fangjia.bkoff.util;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.kafka.core.KafkaTemplate;
|
||||
import org.springframework.kafka.support.SendResult;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
/**
|
||||
* 通用 Kafka 工具类
|
||||
* 提供异步发送 Kafka 消息的功能
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2024/06/21
|
||||
*/
|
||||
@Slf4j
|
||||
@Component // 将其标注为 Spring 管理的组件
|
||||
public class KafkaUtils {
|
||||
|
||||
private final KafkaTemplate<Integer, String> kafkaTemplate;
|
||||
private final ExecutorService executorService; // 用于异步执行
|
||||
|
||||
/**
|
||||
* 构造方法
|
||||
* Spring 会自动注入 KafkaTemplate
|
||||
*
|
||||
* @param kafkaTemplate KafkaTemplate 用于发送消息
|
||||
*/
|
||||
public KafkaUtils(KafkaTemplate<Integer, String> kafkaTemplate) {
|
||||
this.kafkaTemplate = kafkaTemplate;
|
||||
// 初始化一个线程池,使用 10 个线程
|
||||
this.executorService = Executors.newFixedThreadPool(10);
|
||||
}
|
||||
|
||||
/**
|
||||
* 异步发送消息到指定的 Kafka 主题
|
||||
*
|
||||
* @param topic Kafka 主题
|
||||
* @param data 消息内容
|
||||
*/
|
||||
public void sendMessage(final String topic, final String data) {
|
||||
executorService.submit(new Callable<Void>() {
|
||||
@Override
|
||||
public Void call() throws Exception {
|
||||
try {
|
||||
SendResult<Integer, String> sendResult = kafkaTemplate.send(topic, data).get(); // 阻塞直到发送完成
|
||||
log.info("Message sent successfully: {}", sendResult);
|
||||
} catch (Exception ex) {
|
||||
log.error("Kafka send message error = {}, topic = {}", ex.getMessage(), topic);
|
||||
log.error("Data: {}", data);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭线程池
|
||||
* 在程序结束时需要调用该方法以释放资源
|
||||
*/
|
||||
public void shutdown() {
|
||||
executorService.shutdown();
|
||||
}
|
||||
}
|
||||
@@ -77,30 +77,4 @@ merchantno=10000469946
|
||||
#appKey
|
||||
appKey=SQKK10000469946
|
||||
#\u7EDF\u4E00\u9274\u6743\u7ED1\u5361\u8BF7\u6C42
|
||||
authbindcardreqUri=/rest/v1.0/paperorder/unified/pay
|
||||
|
||||
# Kafka bootstrap servers
|
||||
kafka.bootstrap-servers=192.168.0.89:19092,192.168.0.90:19092,192.168.0.91:19092
|
||||
|
||||
# Producer configuration
|
||||
kafka.producer.retries=0
|
||||
kafka.producer.batch-size=16384
|
||||
kafka.producer.buffer-memory=254288
|
||||
kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
|
||||
kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
|
||||
kafka.producer.auto-topic-creation.auto-create=true
|
||||
kafka.producer.properties.linger.ms=1
|
||||
kafka.producer.properties.session.timeout.ms=15000
|
||||
kafka.producer.properties.sasl.mechanism=PLAIN
|
||||
kafka.producer.properties.security.protocol=SASL_PLAINTEXT
|
||||
kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456";
|
||||
|
||||
# Consumer configuration
|
||||
kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
|
||||
kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
|
||||
kafka.consumer.group-id=log-data-group
|
||||
kafka.consumer.auto-offset-reset=earliest
|
||||
kafka.consumer.properties.session.timeout.ms=15000
|
||||
kafka.consumer.properties.sasl.mechanism=PLAIN
|
||||
kafka.consumer.properties.security.protocol=SASL_PLAINTEXT
|
||||
kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456";
|
||||
authbindcardreqUri=/rest/v1.0/paperorder/unified/pay
|
||||
-1
@@ -4215,7 +4215,6 @@ class CommonDatautils {
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
println index
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
// }
|
||||
})
|
||||
|
||||
+123
-100
@@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
for(def cityObj : cityList){
|
||||
// if(cityObj.city == "ÉϺ£ÊÐ"){
|
||||
// if(cityObj.city == "ºúÑîºÓÊÐ"){
|
||||
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
|
||||
println cityObj.city+" "+results.size()
|
||||
def rows = running(cityObj.city, results)
|
||||
save(rows)
|
||||
println cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def rows = running(cityObj.city, results)
|
||||
save(rows)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
@@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
return rets
|
||||
}
|
||||
|
||||
def getStandardRegion(mzbRegionMap, district){
|
||||
@@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
saveObj.name = map.name
|
||||
|
||||
if(map.cat){
|
||||
def arrs = map.cat.split(";|:")
|
||||
if(arrs.size()>=1){
|
||||
saveObj.type = arrs[0]
|
||||
}
|
||||
if(arrs.size()>=2){
|
||||
saveObj.secondary_type = arrs[1]
|
||||
}else{
|
||||
saveObj.secondary_type = saveObj.type
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
def smallstr = arrs[2]
|
||||
def smallArr = null
|
||||
if(smallstr.contains("|")){
|
||||
smallArr = smallstr.split("|")
|
||||
}
|
||||
if(smallArr && smallArr.size()>=1){
|
||||
saveObj.small_type = smallArr[0]
|
||||
}else{
|
||||
saveObj.small_type = smallstr
|
||||
}
|
||||
}else{
|
||||
saveObj.small_type = saveObj.secondary_type
|
||||
}
|
||||
|
||||
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
|
||||
|
||||
saveObj = catWash(saveObj, map)
|
||||
}
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
|
||||
@@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if(location){
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
if(town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
saveObj = addTown(saveObj, towns, location)
|
||||
}
|
||||
|
||||
if(map.address){
|
||||
saveObj = addressWash(saveObj, map, mzbRegionMap, towns)
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def catWash(saveObj, map){
|
||||
def arrs = map.cat.split(";|:")
|
||||
if(arrs.size()>=1){
|
||||
saveObj.type = arrs[0]
|
||||
}
|
||||
if(arrs.size()>=2){
|
||||
saveObj.secondary_type = arrs[1]
|
||||
}else{
|
||||
saveObj.secondary_type = saveObj.type
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
def smallstr = arrs[2]
|
||||
def smallArr = null
|
||||
if(smallstr.contains("|")){
|
||||
smallArr = smallstr.split("|")
|
||||
}
|
||||
if(smallArr && smallArr.size()>=1){
|
||||
saveObj.small_type = smallArr[0]
|
||||
}else{
|
||||
saveObj.small_type = smallstr
|
||||
}
|
||||
}else{
|
||||
saveObj.small_type = saveObj.secondary_type
|
||||
}
|
||||
|
||||
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def addTown(saveObj, towns, location){
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
if(town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(map.address && !saveObj.township){
|
||||
// for(def town : towns){
|
||||
// if(map.address.contains(town.street)){
|
||||
@@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
// println map.address +" º¬½ÖµÀ,Õò£¬Ïç"
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
if(map.address){
|
||||
if(map.address.length()<3){
|
||||
saveObj.address = ""
|
||||
}else{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
while(i<5){
|
||||
if(address && address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(map.province.toString().length())
|
||||
}
|
||||
if(address && address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(map.city.toString().length())
|
||||
}
|
||||
def addressWash(saveObj, map, mzbRegionMap, towns){
|
||||
if(map.address.length()<3){
|
||||
saveObj.address = ""
|
||||
}else{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
while(i<5){
|
||||
if(address && address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(map.province.toString().length())
|
||||
}
|
||||
if(address && address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(map.city.toString().length())
|
||||
}
|
||||
|
||||
if(address && saveObj.district){
|
||||
def tjjObj = mzbRegionMap.get(saveObj.district)
|
||||
if(tjjObj){
|
||||
def alias = tjjObj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
if(address && saveObj.district){
|
||||
def tjjObj = mzbRegionMap.get(saveObj.district)
|
||||
if(tjjObj){
|
||||
def alias = null
|
||||
if(tjjObj.alias){
|
||||
alias = new CopyOnWriteArrayList<>(tjjObj.alias);
|
||||
}else{
|
||||
alias = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
if(!alias.contains(saveObj.district)){
|
||||
alias << saveObj.district
|
||||
}
|
||||
if(alias.size()>=2){
|
||||
alias = alias.sort(){ a, b ->
|
||||
b.length().compareTo(a.length());
|
||||
}
|
||||
if(!alias.contains(saveObj.district)){
|
||||
alias << saveObj.district
|
||||
}
|
||||
// if(alias.size()>=2){
|
||||
// alias.sort(){ a, b ->
|
||||
// b.length().compareTo(a.length());
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
for(def regionName : alias){
|
||||
if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){
|
||||
address = address.substring(regionName.toString().length())
|
||||
break
|
||||
}
|
||||
for(def regionName : alias){
|
||||
if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){
|
||||
address = address.substring(regionName.toString().length())
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(address && address.toString().startsWith("ÊÐÏ½Çø")){
|
||||
address = address.toString().substring(3)
|
||||
}
|
||||
if(address && address.toString().startsWith("ÊÐÏ½Çø")){
|
||||
address = address.toString().substring(3)
|
||||
}
|
||||
|
||||
if(address && towns){
|
||||
for(def town : towns){
|
||||
def townAlias = town.alias
|
||||
if(!townAlias){
|
||||
townAlias = []
|
||||
if(address && towns){
|
||||
for(def town : towns){
|
||||
def townAlias = null
|
||||
if(town.alias){
|
||||
townAlias = new CopyOnWriteArrayList<>(town.alias);
|
||||
}else{
|
||||
townAlias = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
if(!townAlias.contains(town.street)){
|
||||
townAlias << town.street
|
||||
}
|
||||
if(townAlias.size()>=2){
|
||||
townAlias = townAlias.sort(){ a, b ->
|
||||
b.length().compareTo(a.length());
|
||||
}
|
||||
}
|
||||
if(!townAlias.contains(town.street)){
|
||||
townAlias << town.street
|
||||
}
|
||||
// if(townAlias.size()>=2){
|
||||
// townAlias = townAlias.sort(){ a, b ->
|
||||
// b.length().compareTo(a.length());
|
||||
// }
|
||||
// }
|
||||
|
||||
for(def townName : townAlias){
|
||||
if(address.startsWith(townName)){
|
||||
address = address.substring(townName.toString().length())
|
||||
for(def townName : townAlias){
|
||||
if(address.startsWith(townName)){
|
||||
address = address.substring(townName.toString().length())
|
||||
// if(!saveObj.township){
|
||||
// saveObj.township = town.street
|
||||
// }
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
if(address && address.startsWith("(") && address.endsWith(")")){
|
||||
saveObj.address = address.substring(1, address.length()-1)
|
||||
}else{
|
||||
saveObj.address = address
|
||||
}
|
||||
i++
|
||||
}
|
||||
if(address && address.startsWith("(") && address.endsWith(")")){
|
||||
saveObj.address = address.substring(1, address.length()-1)
|
||||
}else{
|
||||
saveObj.address = address
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
@@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int batchSize = 2000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class PoiNineSplitBatch extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PoiNineSplitBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
def run(){
|
||||
def i = 0;
|
||||
while(i<360) {
|
||||
def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\new_poi\\new_poi_address_split_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if (d['C'] && !d['G']) {
|
||||
println d['C'] +" "+ d['E']
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E'])
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
|
||||
d['G'] = jsonObject.get("street")
|
||||
}
|
||||
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
|
||||
d['H'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
|
||||
d['I'] = jsonObject.get("township")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
println fileName +" 九级拆分完成"
|
||||
}else{
|
||||
println fileName +" 文件不存在"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user