From ce66ee421302d0f598e4013183ae0c9530862783 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 27 Nov 2024 18:22:14 +0800 Subject: [PATCH 1/4] update --- .../script/commData/CommonDatautils.groovy | 1 - .../gj/data_handle/NewGaoDePoiBatch.groovy | 223 ++++++++++-------- 2 files changed, 123 insertions(+), 101 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ed3314d22..9db36e3d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4215,7 +4215,6 @@ class CommonDatautils { ElasticUtils.executeSqlQuery(sql, { map-> // if(map.address && map.address.contains("五角场")){ index++ - println index rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 15b8005d6..822eee441 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{ def cityList = CommonDatautils.findFullNameCitys().reverse() for(def cityObj : cityList){ -// if(cityObj.city == "上海市"){ +// if(cityObj.city == "胡杨河市"){ def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) - println cityObj.city+" "+results.size() - def rows = running(cityObj.city, results) - save(rows) + println cityObj.province+" " +cityObj.city+" "+results.size() + if(results){ + def rows = running(cityObj.city, results) + save(rows) + } // } } } @@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{ }); } latch.await() - return rets; + return rets } def getStandardRegion(mzbRegionMap, district){ @@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{ } saveObj.name = map.name + if(map.cat){ - def arrs = map.cat.split(";|:") - if(arrs.size()>=1){ - saveObj.type = arrs[0] - } - if(arrs.size()>=2){ - saveObj.secondary_type = arrs[1] - }else{ - saveObj.secondary_type = saveObj.type - } - if(arrs.size()>=3){ - def smallstr = arrs[2] - def smallArr = null - if(smallstr.contains("|")){ - smallArr = smallstr.split("|") - } - if(smallArr && smallArr.size()>=1){ - saveObj.small_type = smallArr[0] - }else{ - saveObj.small_type = smallstr - } - }else{ - saveObj.small_type = saveObj.secondary_type - } - - saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type - + saveObj = catWash(saveObj, map) } def location = [:] location.lon = DataHandleUtils.getDoubleRound(6, map.lng) @@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{ def towns = townMap.get(saveObj.district) if(location){ - if(towns){ - for(def town : towns){ - if(town.shape && town.shape.coordinates){ - def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) - if(flag){ - saveObj.township = town.street - break - } + saveObj = addTown(saveObj, towns, location) + } + + if(map.address){ + saveObj = addressWash(saveObj, map, mzbRegionMap, towns) + } + return saveObj + } + + def catWash(saveObj, map){ + def arrs = map.cat.split(";|:") + if(arrs.size()>=1){ + saveObj.type = arrs[0] + } + if(arrs.size()>=2){ + saveObj.secondary_type = arrs[1] + }else{ + saveObj.secondary_type = saveObj.type + } + if(arrs.size()>=3){ + def smallstr = arrs[2] + def smallArr = null + if(smallstr.contains("|")){ + smallArr = smallstr.split("|") + } + if(smallArr && smallArr.size()>=1){ + saveObj.small_type = smallArr[0] + }else{ + saveObj.small_type = smallstr + } + }else{ + saveObj.small_type = saveObj.secondary_type + } + + saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type + return saveObj + } + + def addTown(saveObj, towns, location){ + if(towns){ + for(def town : towns){ + if(town.shape && town.shape.coordinates){ + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + saveObj.township = town.street + break } } + } // if(map.address && !saveObj.township){ // for(def town : towns){ // if(map.address.contains(town.street)){ @@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{ // println map.address +" 含街道,镇,乡" // } // } - } } + return saveObj + } - if(map.address){ - if(map.address.length()<3){ - saveObj.address = "" - }else{ - def address = map.address - def i = 0 - while(i<5){ - if(address && address.toString().startsWith(map.province)){ - address = address.toString().substring(map.province.toString().length()) - } - if(address && address.toString().startsWith(map.city)){ - address = address.toString().substring(map.city.toString().length()) - } + def addressWash(saveObj, map, mzbRegionMap, towns){ + if(map.address.length()<3){ + saveObj.address = "" + }else{ + def address = map.address + def i = 0 + while(i<5){ + if(address && address.toString().startsWith(map.province)){ + address = address.toString().substring(map.province.toString().length()) + } + if(address && address.toString().startsWith(map.city)){ + address = address.toString().substring(map.city.toString().length()) + } - if(address && saveObj.district){ - def tjjObj = mzbRegionMap.get(saveObj.district) - if(tjjObj){ - def alias = tjjObj.alias - if(!alias){ - alias = [] + if(address && saveObj.district){ + def tjjObj = mzbRegionMap.get(saveObj.district) + if(tjjObj){ + def alias = null + if(tjjObj.alias){ + alias = new CopyOnWriteArrayList<>(tjjObj.alias); + }else{ + alias = new CopyOnWriteArrayList<>(); + } + + if(!alias.contains(saveObj.district)){ + alias << saveObj.district + } + if(alias.size()>=2){ + alias = alias.sort(){ a, b -> + b.length().compareTo(a.length()); } - if(!alias.contains(saveObj.district)){ - alias << saveObj.district - } -// if(alias.size()>=2){ -// alias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } + } - for(def regionName : alias){ - if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ - address = address.substring(regionName.toString().length()) - break - } + for(def regionName : alias){ + if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ + address = address.substring(regionName.toString().length()) + break } } } + } - if(address && address.toString().startsWith("市辖区")){ - address = address.toString().substring(3) - } + if(address && address.toString().startsWith("市辖区")){ + address = address.toString().substring(3) + } - if(address && towns){ - for(def town : towns){ - def townAlias = town.alias - if(!townAlias){ - townAlias = [] + if(address && towns){ + for(def town : towns){ + def townAlias = null + if(town.alias){ + townAlias = new CopyOnWriteArrayList<>(town.alias); + }else{ + townAlias = new CopyOnWriteArrayList<>(); + } + + if(!townAlias.contains(town.street)){ + townAlias << town.street + } + if(townAlias.size()>=2){ + townAlias = townAlias.sort(){ a, b -> + b.length().compareTo(a.length()); + } } - if(!townAlias.contains(town.street)){ - townAlias << town.street - } -// if(townAlias.size()>=2){ -// townAlias = townAlias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } - for(def townName : townAlias){ - if(address.startsWith(townName)){ - address = address.substring(townName.toString().length()) + for(def townName : townAlias){ + if(address.startsWith(townName)){ + address = address.substring(townName.toString().length()) // if(!saveObj.township){ // saveObj.township = town.street // } - break - } + break } } } - i++ - } - if(address && address.startsWith("(") && address.endsWith(")")){ - saveObj.address = address.substring(1, address.length()-1) - }else{ - saveObj.address = address } + i++ + } + if(address && address.startsWith("(") && address.endsWith(")")){ + saveObj.address = address.substring(1, address.length()-1) + }else{ + saveObj.address = address } } return saveObj @@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def save(rows){ BulkRequest request = new BulkRequest(); - int batchSize = 1000; + int batchSize = 2000; int processedCount = 0; def client = ElasticUtils89.getClient("192.168.0.89") From 1a537605f6c69d1b64f7b9e6da25ccf54d471ef5 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 28 Nov 2024 10:21:41 +0800 Subject: [PATCH 2/4] =?UTF-8?q?Kafka=E9=82=84=E5=8E=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-bkoff/pom.xml | 11 --- .../bkoff/controllers/DistrictController.java | 18 ----- .../com/fangjia/bkoff/util/KafkaUtils.java | 67 ------------------- .../main/resources/fangjia-bkoff.properties | 28 +------- 4 files changed, 1 insertion(+), 123 deletions(-) delete mode 100644 projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java diff --git a/projects/fangjia-bkoff/pom.xml b/projects/fangjia-bkoff/pom.xml index 6c40448a0..69e2d3b60 100644 --- a/projects/fangjia-bkoff/pom.xml +++ b/projects/fangjia-bkoff/pom.xml @@ -183,17 +183,6 @@ easyexcel 4.0.1 - - org.springframework.kafka - spring-kafka - 2.8.9 - - - org.projectlombok - lombok - 1.18.28 - provided - diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java index bc33eddc6..e57a129f7 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java @@ -1,11 +1,7 @@ package com.fangjia.bkoff.controllers; -import com.fangjia.bkoff.model.DistrictJsonModel; import com.fangjia.bkoff.model.DistrictModel; import com.fangjia.bkoff.model.JsonModel; -import com.fangjia.bkoff.util.AddressParser; -import com.fangjia.bkoff.util.GeocodeUtils; -import com.fangjia.bkoff.util.KafkaUtils; import com.fangjia.core.Constants; import com.fangjia.core.RegionData; import com.fangjia.core.util.*; @@ -90,8 +86,6 @@ public class DistrictController { private DealPriceService dealPriceService; private AverageStatisticsService averageStatisticsService; - private KafkaUtils kafkaUtils; - private static List distTypes = Arrays.asList("多层", "小高层", "超高层", "高层", "花园洋房", "老洋房", "四合院"); private static List villaTypes = Arrays.asList("独栋", "双拼", "联排", "叠加"); @@ -2288,14 +2282,6 @@ public class DistrictController { data.put("markMap", markMap); data.put("modificationDate", new Date()); districtService.updateById(data, district.getId()); - JSONObject massage = new JSONObject(); - massage.put("city",district.getCity()); - massage.put("region",district.getRegion()); - massage.put("name",district.getName()); - massage.put("estateType2",district.getEstateType2()); - massage.put("id",district.getId()); - massage.put("type",2); - kafkaUtils.sendMessage("seaweed-data-topic", massage.toString()); //创建更新任务,用为更新常州农商行数据 createJnTask(district.getId(), null); msg = "保存成功了!!!"; @@ -4222,8 +4208,4 @@ public class DistrictController { public void setAverageStatisticsService(AverageStatisticsService averageStatisticsService) { this.averageStatisticsService = averageStatisticsService; } - - public void setKafkaUtils(KafkaUtils kafkaUtils) { - this.kafkaUtils = kafkaUtils; - } } diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java deleted file mode 100644 index 31a48baa3..000000000 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.fangjia.bkoff.util; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.SendResult; -import org.springframework.stereotype.Component; - -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * 通用 Kafka 工具类 - * 提供异步发送 Kafka 消息的功能 - * - * @author shenjian - * @since 2024/06/21 - */ -@Slf4j -@Component // 将其标注为 Spring 管理的组件 -public class KafkaUtils { - - private final KafkaTemplate kafkaTemplate; - private final ExecutorService executorService; // 用于异步执行 - - /** - * 构造方法 - * Spring 会自动注入 KafkaTemplate - * - * @param kafkaTemplate KafkaTemplate 用于发送消息 - */ - public KafkaUtils(KafkaTemplate kafkaTemplate) { - this.kafkaTemplate = kafkaTemplate; - // 初始化一个线程池,使用 10 个线程 - this.executorService = Executors.newFixedThreadPool(10); - } - - /** - * 异步发送消息到指定的 Kafka 主题 - * - * @param topic Kafka 主题 - * @param data 消息内容 - */ - public void sendMessage(final String topic, final String data) { - executorService.submit(new Callable() { - @Override - public Void call() throws Exception { - try { - SendResult sendResult = kafkaTemplate.send(topic, data).get(); // 阻塞直到发送完成 - log.info("Message sent successfully: {}", sendResult); - } catch (Exception ex) { - log.error("Kafka send message error = {}, topic = {}", ex.getMessage(), topic); - log.error("Data: {}", data); - } - return null; - } - }); - } - - /** - * 关闭线程池 - * 在程序结束时需要调用该方法以释放资源 - */ - public void shutdown() { - executorService.shutdown(); - } -} diff --git a/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties b/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties index bcc1ddcf9..99a3432fe 100644 --- a/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties +++ b/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties @@ -77,30 +77,4 @@ merchantno=10000469946 #appKey appKey=SQKK10000469946 #\u7EDF\u4E00\u9274\u6743\u7ED1\u5361\u8BF7\u6C42 -authbindcardreqUri=/rest/v1.0/paperorder/unified/pay - -# Kafka bootstrap servers -kafka.bootstrap-servers=192.168.0.89:19092,192.168.0.90:19092,192.168.0.91:19092 - -# Producer configuration -kafka.producer.retries=0 -kafka.producer.batch-size=16384 -kafka.producer.buffer-memory=254288 -kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer -kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer -kafka.producer.auto-topic-creation.auto-create=true -kafka.producer.properties.linger.ms=1 -kafka.producer.properties.session.timeout.ms=15000 -kafka.producer.properties.sasl.mechanism=PLAIN -kafka.producer.properties.security.protocol=SASL_PLAINTEXT -kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456"; - -# Consumer configuration -kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer -kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer -kafka.consumer.group-id=log-data-group -kafka.consumer.auto-offset-reset=earliest -kafka.consumer.properties.session.timeout.ms=15000 -kafka.consumer.properties.sasl.mechanism=PLAIN -kafka.consumer.properties.security.protocol=SASL_PLAINTEXT -kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456"; +authbindcardreqUri=/rest/v1.0/paperorder/unified/pay \ No newline at end of file From 6c238241293df99b8f3d5517147e476f0238d870 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 28 Nov 2024 10:25:25 +0800 Subject: [PATCH 3/4] =?UTF-8?q?Kafka=E9=82=84=E5=8E=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/bkoff/controllers/DistrictController.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java index e57a129f7..a9c2fb20d 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java @@ -1,7 +1,10 @@ package com.fangjia.bkoff.controllers; +import com.fangjia.bkoff.model.DistrictJsonModel; import com.fangjia.bkoff.model.DistrictModel; import com.fangjia.bkoff.model.JsonModel; +import com.fangjia.bkoff.util.AddressParser; +import com.fangjia.bkoff.util.GeocodeUtils; import com.fangjia.core.Constants; import com.fangjia.core.RegionData; import com.fangjia.core.util.*; From f8bc248feb94baf97d78f2a7c47cffe6fa9f1d8c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 28 Nov 2024 14:03:53 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=96=B0poi=E4=B9=9D=E7=BA=A7=E6=8B=86?= =?UTF-8?q?=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/PoiNineSplitBatch.groovy | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy new file mode 100644 index 000000000..c5ea6d76d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy @@ -0,0 +1,54 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils + +@BatchOwner(Watcher.GaoJuan) +class PoiNineSplitBatch extends BaseBatch{ + + static main(args) { + runBatch(new PoiNineSplitBatch(), args) + } + + @Override + void process(Object args) { + run() + } + def run(){ + def i = 0; + while(i<360) { + def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\new_poi\\new_poi_address_split_${i}.xlsx" + } + + File file = new File(fileName); + if (file.exists()) { + ExcelUtils.multiThreadingModify(fileName, { d -> + if (d['C'] && !d['G']) { + println d['C'] +" "+ d['E'] + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E']) + if (jsonObject) { + if (jsonObject.get("street") && jsonObject.get("street") != "null") { + d['G'] = jsonObject.get("street") + } + if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") { + d['H'] = jsonObject.get("streetNumber") + } + if (jsonObject.get("township") && jsonObject.get("township") != "null") { + d['I'] = jsonObject.get("township") + } + } + } + }) + println fileName +" 九级拆分完成" + }else{ + println fileName +" 文件不存在" + } + i++ + } + } +}