diff --git a/projects/fangjia-bkoff/pom.xml b/projects/fangjia-bkoff/pom.xml
index 6c40448a0..69e2d3b60 100644
--- a/projects/fangjia-bkoff/pom.xml
+++ b/projects/fangjia-bkoff/pom.xml
@@ -183,17 +183,6 @@
easyexcel
4.0.1
-
- org.springframework.kafka
- spring-kafka
- 2.8.9
-
-
- org.projectlombok
- lombok
- 1.18.28
- provided
-
diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java
index bc33eddc6..a9c2fb20d 100644
--- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java
+++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java
@@ -5,7 +5,6 @@ import com.fangjia.bkoff.model.DistrictModel;
import com.fangjia.bkoff.model.JsonModel;
import com.fangjia.bkoff.util.AddressParser;
import com.fangjia.bkoff.util.GeocodeUtils;
-import com.fangjia.bkoff.util.KafkaUtils;
import com.fangjia.core.Constants;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.*;
@@ -90,8 +89,6 @@ public class DistrictController {
private DealPriceService dealPriceService;
private AverageStatisticsService averageStatisticsService;
- private KafkaUtils kafkaUtils;
-
private static List distTypes = Arrays.asList("¶à²ã", "С¸ß²ã", "³¬¸ß²ã", "¸ß²ã", "»¨Ô°Ñó·¿", "ÀÏÑó·¿", "ËĺÏÔº");
private static List villaTypes = Arrays.asList("¶À¶°", "˫ƴ", "ÁªÅÅ", "µþ¼Ó");
@@ -2288,14 +2285,6 @@ public class DistrictController {
data.put("markMap", markMap);
data.put("modificationDate", new Date());
districtService.updateById(data, district.getId());
- JSONObject massage = new JSONObject();
- massage.put("city",district.getCity());
- massage.put("region",district.getRegion());
- massage.put("name",district.getName());
- massage.put("estateType2",district.getEstateType2());
- massage.put("id",district.getId());
- massage.put("type",2);
- kafkaUtils.sendMessage("seaweed-data-topic", massage.toString());
//´´½¨¸üÐÂÈÎÎñ£¬ÓÃΪ¸üг£ÖÝÅ©ÉÌÐÐÊý¾Ý
createJnTask(district.getId(), null);
msg = "±£´æ³É¹¦ÁË£¡£¡£¡";
@@ -4222,8 +4211,4 @@ public class DistrictController {
public void setAverageStatisticsService(AverageStatisticsService averageStatisticsService) {
this.averageStatisticsService = averageStatisticsService;
}
-
- public void setKafkaUtils(KafkaUtils kafkaUtils) {
- this.kafkaUtils = kafkaUtils;
- }
}
diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java
deleted file mode 100644
index 31a48baa3..000000000
--- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.fangjia.bkoff.util;
-
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.kafka.core.KafkaTemplate;
-import org.springframework.kafka.support.SendResult;
-import org.springframework.stereotype.Component;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * ͨÓà Kafka ¹¤¾ßÀà
- * ÌṩÒì²½·¢ËÍ Kafka ÏûÏ¢µÄ¹¦ÄÜ
- *
- * @author shenjian
- * @since 2024/06/21
- */
-@Slf4j
-@Component // ½«Æä±êעΪ Spring ¹ÜÀíµÄ×é¼þ
-public class KafkaUtils {
-
- private final KafkaTemplate kafkaTemplate;
- private final ExecutorService executorService; // ÓÃÓÚÒì²½Ö´ÐÐ
-
- /**
- * ¹¹Ôì·½·¨
- * Spring »á×Ô¶¯×¢Èë KafkaTemplate
- *
- * @param kafkaTemplate KafkaTemplate ÓÃÓÚ·¢ËÍÏûÏ¢
- */
- public KafkaUtils(KafkaTemplate kafkaTemplate) {
- this.kafkaTemplate = kafkaTemplate;
- // ³õʼ»¯Ò»¸öÏ̳߳أ¬Ê¹Óà 10 ¸öÏß³Ì
- this.executorService = Executors.newFixedThreadPool(10);
- }
-
- /**
- * Òì²½·¢ËÍÏûÏ¢µ½Ö¸¶¨µÄ Kafka Ö÷Ìâ
- *
- * @param topic Kafka Ö÷Ìâ
- * @param data ÏûÏ¢ÄÚÈÝ
- */
- public void sendMessage(final String topic, final String data) {
- executorService.submit(new Callable() {
- @Override
- public Void call() throws Exception {
- try {
- SendResult sendResult = kafkaTemplate.send(topic, data).get(); // ×èÈûÖ±µ½·¢ËÍÍê³É
- log.info("Message sent successfully: {}", sendResult);
- } catch (Exception ex) {
- log.error("Kafka send message error = {}, topic = {}", ex.getMessage(), topic);
- log.error("Data: {}", data);
- }
- return null;
- }
- });
- }
-
- /**
- * ¹Ø±ÕÏ̳߳Ø
- * ÔÚ³ÌÐò½áÊøÊ±ÐèÒªµ÷Óø÷½·¨ÒÔÊÍ·Å×ÊÔ´
- */
- public void shutdown() {
- executorService.shutdown();
- }
-}
diff --git a/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties b/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties
index bcc1ddcf9..99a3432fe 100644
--- a/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties
+++ b/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties
@@ -77,30 +77,4 @@ merchantno=10000469946
#appKey
appKey=SQKK10000469946
#\u7EDF\u4E00\u9274\u6743\u7ED1\u5361\u8BF7\u6C42
-authbindcardreqUri=/rest/v1.0/paperorder/unified/pay
-
-# Kafka bootstrap servers
-kafka.bootstrap-servers=192.168.0.89:19092,192.168.0.90:19092,192.168.0.91:19092
-
-# Producer configuration
-kafka.producer.retries=0
-kafka.producer.batch-size=16384
-kafka.producer.buffer-memory=254288
-kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
-kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
-kafka.producer.auto-topic-creation.auto-create=true
-kafka.producer.properties.linger.ms=1
-kafka.producer.properties.session.timeout.ms=15000
-kafka.producer.properties.sasl.mechanism=PLAIN
-kafka.producer.properties.security.protocol=SASL_PLAINTEXT
-kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456";
-
-# Consumer configuration
-kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
-kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
-kafka.consumer.group-id=log-data-group
-kafka.consumer.auto-offset-reset=earliest
-kafka.consumer.properties.session.timeout.ms=15000
-kafka.consumer.properties.sasl.mechanism=PLAIN
-kafka.consumer.properties.security.protocol=SASL_PLAINTEXT
-kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456";
+authbindcardreqUri=/rest/v1.0/paperorder/unified/pay
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index ed3314d22..9db36e3d8 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -4215,7 +4215,6 @@ class CommonDatautils {
ElasticUtils.executeSqlQuery(sql, { map->
// if(map.address && map.address.contains("Îå½Ç³¡")){
index++
- println index
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
// }
})
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy
index 15b8005d6..822eee441 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy
@@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{
def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){
-// if(cityObj.city == "ÉϺ£ÊÐ"){
+// if(cityObj.city == "ºúÑîºÓÊÐ"){
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
- println cityObj.city+" "+results.size()
- def rows = running(cityObj.city, results)
- save(rows)
+ println cityObj.province+" " +cityObj.city+" "+results.size()
+ if(results){
+ def rows = running(cityObj.city, results)
+ save(rows)
+ }
// }
}
}
@@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{
});
}
latch.await()
- return rets;
+ return rets
}
def getStandardRegion(mzbRegionMap, district){
@@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{
}
saveObj.name = map.name
+
if(map.cat){
- def arrs = map.cat.split(";|:")
- if(arrs.size()>=1){
- saveObj.type = arrs[0]
- }
- if(arrs.size()>=2){
- saveObj.secondary_type = arrs[1]
- }else{
- saveObj.secondary_type = saveObj.type
- }
- if(arrs.size()>=3){
- def smallstr = arrs[2]
- def smallArr = null
- if(smallstr.contains("|")){
- smallArr = smallstr.split("|")
- }
- if(smallArr && smallArr.size()>=1){
- saveObj.small_type = smallArr[0]
- }else{
- saveObj.small_type = smallstr
- }
- }else{
- saveObj.small_type = saveObj.secondary_type
- }
-
- saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
-
+ saveObj = catWash(saveObj, map)
}
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
@@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{
def towns = townMap.get(saveObj.district)
if(location){
- if(towns){
- for(def town : towns){
- if(town.shape && town.shape.coordinates){
- def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
- if(flag){
- saveObj.township = town.street
- break
- }
+ saveObj = addTown(saveObj, towns, location)
+ }
+
+ if(map.address){
+ saveObj = addressWash(saveObj, map, mzbRegionMap, towns)
+ }
+ return saveObj
+ }
+
+ def catWash(saveObj, map){
+ def arrs = map.cat.split(";|:")
+ if(arrs.size()>=1){
+ saveObj.type = arrs[0]
+ }
+ if(arrs.size()>=2){
+ saveObj.secondary_type = arrs[1]
+ }else{
+ saveObj.secondary_type = saveObj.type
+ }
+ if(arrs.size()>=3){
+ def smallstr = arrs[2]
+ def smallArr = null
+ if(smallstr.contains("|")){
+ smallArr = smallstr.split("|")
+ }
+ if(smallArr && smallArr.size()>=1){
+ saveObj.small_type = smallArr[0]
+ }else{
+ saveObj.small_type = smallstr
+ }
+ }else{
+ saveObj.small_type = saveObj.secondary_type
+ }
+
+ saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
+ return saveObj
+ }
+
+ def addTown(saveObj, towns, location){
+ if(towns){
+ for(def town : towns){
+ if(town.shape && town.shape.coordinates){
+ def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
+ if(flag){
+ saveObj.township = town.street
+ break
}
}
+ }
// if(map.address && !saveObj.township){
// for(def town : towns){
// if(map.address.contains(town.street)){
@@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{
// println map.address +" º¬½ÖµÀ,Õò£¬Ïç"
// }
// }
- }
}
+ return saveObj
+ }
- if(map.address){
- if(map.address.length()<3){
- saveObj.address = ""
- }else{
- def address = map.address
- def i = 0
- while(i<5){
- if(address && address.toString().startsWith(map.province)){
- address = address.toString().substring(map.province.toString().length())
- }
- if(address && address.toString().startsWith(map.city)){
- address = address.toString().substring(map.city.toString().length())
- }
+ def addressWash(saveObj, map, mzbRegionMap, towns){
+ if(map.address.length()<3){
+ saveObj.address = ""
+ }else{
+ def address = map.address
+ def i = 0
+ while(i<5){
+ if(address && address.toString().startsWith(map.province)){
+ address = address.toString().substring(map.province.toString().length())
+ }
+ if(address && address.toString().startsWith(map.city)){
+ address = address.toString().substring(map.city.toString().length())
+ }
- if(address && saveObj.district){
- def tjjObj = mzbRegionMap.get(saveObj.district)
- if(tjjObj){
- def alias = tjjObj.alias
- if(!alias){
- alias = []
+ if(address && saveObj.district){
+ def tjjObj = mzbRegionMap.get(saveObj.district)
+ if(tjjObj){
+ def alias = null
+ if(tjjObj.alias){
+ alias = new CopyOnWriteArrayList<>(tjjObj.alias);
+ }else{
+ alias = new CopyOnWriteArrayList<>();
+ }
+
+ if(!alias.contains(saveObj.district)){
+ alias << saveObj.district
+ }
+ if(alias.size()>=2){
+ alias = alias.sort(){ a, b ->
+ b.length().compareTo(a.length());
}
- if(!alias.contains(saveObj.district)){
- alias << saveObj.district
- }
-// if(alias.size()>=2){
-// alias.sort(){ a, b ->
-// b.length().compareTo(a.length());
-// }
-// }
+ }
- for(def regionName : alias){
- if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){
- address = address.substring(regionName.toString().length())
- break
- }
+ for(def regionName : alias){
+ if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){
+ address = address.substring(regionName.toString().length())
+ break
}
}
}
+ }
- if(address && address.toString().startsWith("ÊÐÏ½Çø")){
- address = address.toString().substring(3)
- }
+ if(address && address.toString().startsWith("ÊÐÏ½Çø")){
+ address = address.toString().substring(3)
+ }
- if(address && towns){
- for(def town : towns){
- def townAlias = town.alias
- if(!townAlias){
- townAlias = []
+ if(address && towns){
+ for(def town : towns){
+ def townAlias = null
+ if(town.alias){
+ townAlias = new CopyOnWriteArrayList<>(town.alias);
+ }else{
+ townAlias = new CopyOnWriteArrayList<>();
+ }
+
+ if(!townAlias.contains(town.street)){
+ townAlias << town.street
+ }
+ if(townAlias.size()>=2){
+ townAlias = townAlias.sort(){ a, b ->
+ b.length().compareTo(a.length());
+ }
}
- if(!townAlias.contains(town.street)){
- townAlias << town.street
- }
-// if(townAlias.size()>=2){
-// townAlias = townAlias.sort(){ a, b ->
-// b.length().compareTo(a.length());
-// }
-// }
- for(def townName : townAlias){
- if(address.startsWith(townName)){
- address = address.substring(townName.toString().length())
+ for(def townName : townAlias){
+ if(address.startsWith(townName)){
+ address = address.substring(townName.toString().length())
// if(!saveObj.township){
// saveObj.township = town.street
// }
- break
- }
+ break
}
}
}
- i++
- }
- if(address && address.startsWith("(") && address.endsWith(")")){
- saveObj.address = address.substring(1, address.length()-1)
- }else{
- saveObj.address = address
}
+ i++
+ }
+ if(address && address.startsWith("(") && address.endsWith(")")){
+ saveObj.address = address.substring(1, address.length()-1)
+ }else{
+ saveObj.address = address
}
}
return saveObj
@@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{
def save(rows){
BulkRequest request = new BulkRequest();
- int batchSize = 1000;
+ int batchSize = 2000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy
new file mode 100644
index 000000000..c5ea6d76d
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy
@@ -0,0 +1,54 @@
+package com.fangjia.script.gj.data_handle
+
+import com.fangjia.script.BaseBatch
+import com.fangjia.script.BatchOwner
+import com.fangjia.script.Watcher
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.util.ExcelUtils
+
+@BatchOwner(Watcher.GaoJuan)
+class PoiNineSplitBatch extends BaseBatch{
+
+ static main(args) {
+ runBatch(new PoiNineSplitBatch(), args)
+ }
+
+ @Override
+ void process(Object args) {
+ run()
+ }
+ def run(){
+ def i = 0;
+ while(i<360) {
+ def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx"
+ if (CommonDatautils.isWindow()) {
+ fileName = "D:\\Êý¾Ý\\new_poi\\new_poi_address_split_${i}.xlsx"
+ }
+
+ File file = new File(fileName);
+ if (file.exists()) {
+ ExcelUtils.multiThreadingModify(fileName, { d ->
+ if (d['C'] && !d['G']) {
+ println d['C'] +" "+ d['E']
+ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E'])
+ if (jsonObject) {
+ if (jsonObject.get("street") && jsonObject.get("street") != "null") {
+ d['G'] = jsonObject.get("street")
+ }
+ if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
+ d['H'] = jsonObject.get("streetNumber")
+ }
+ if (jsonObject.get("township") && jsonObject.get("township") != "null") {
+ d['I'] = jsonObject.get("township")
+ }
+ }
+ }
+ })
+ println fileName +" ¾Å¼¶²ð·ÖÍê³É"
+ }else{
+ println fileName +" Îļþ²»´æÔÚ"
+ }
+ i++
+ }
+ }
+}