diff --git a/projects/fangjia-bkoff/pom.xml b/projects/fangjia-bkoff/pom.xml index 6c40448a0..69e2d3b60 100644 --- a/projects/fangjia-bkoff/pom.xml +++ b/projects/fangjia-bkoff/pom.xml @@ -183,17 +183,6 @@ easyexcel 4.0.1 - - org.springframework.kafka - spring-kafka - 2.8.9 - - - org.projectlombok - lombok - 1.18.28 - provided - diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java index bc33eddc6..a9c2fb20d 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java @@ -5,7 +5,6 @@ import com.fangjia.bkoff.model.DistrictModel; import com.fangjia.bkoff.model.JsonModel; import com.fangjia.bkoff.util.AddressParser; import com.fangjia.bkoff.util.GeocodeUtils; -import com.fangjia.bkoff.util.KafkaUtils; import com.fangjia.core.Constants; import com.fangjia.core.RegionData; import com.fangjia.core.util.*; @@ -90,8 +89,6 @@ public class DistrictController { private DealPriceService dealPriceService; private AverageStatisticsService averageStatisticsService; - private KafkaUtils kafkaUtils; - private static List distTypes = Arrays.asList("¶à²ã", "С¸ß²ã", "³¬¸ß²ã", "¸ß²ã", "»¨Ô°Ñó·¿", "ÀÏÑó·¿", "ËĺÏÔº"); private static List villaTypes = Arrays.asList("¶À¶°", "˫ƴ", "ÁªÅÅ", "µþ¼Ó"); @@ -2288,14 +2285,6 @@ public class DistrictController { data.put("markMap", markMap); data.put("modificationDate", new Date()); districtService.updateById(data, district.getId()); - JSONObject massage = new JSONObject(); - massage.put("city",district.getCity()); - massage.put("region",district.getRegion()); - massage.put("name",district.getName()); - massage.put("estateType2",district.getEstateType2()); - massage.put("id",district.getId()); - massage.put("type",2); - kafkaUtils.sendMessage("seaweed-data-topic", massage.toString()); //´´½¨¸üÐÂÈÎÎñ£¬ÓÃΪ¸üг£ÖÝÅ©ÉÌÐÐÊý¾Ý createJnTask(district.getId(), null); msg = "±£´æ³É¹¦ÁË£¡£¡£¡"; @@ -4222,8 +4211,4 @@ public class DistrictController { public void setAverageStatisticsService(AverageStatisticsService averageStatisticsService) { this.averageStatisticsService = averageStatisticsService; } - - public void setKafkaUtils(KafkaUtils kafkaUtils) { - this.kafkaUtils = kafkaUtils; - } } diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java deleted file mode 100644 index 31a48baa3..000000000 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/KafkaUtils.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.fangjia.bkoff.util; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.SendResult; -import org.springframework.stereotype.Component; - -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * ͨÓà Kafka ¹¤¾ßÀà - * ÌṩÒì²½·¢ËÍ Kafka ÏûÏ¢µÄ¹¦ÄÜ - * - * @author shenjian - * @since 2024/06/21 - */ -@Slf4j -@Component // ½«Æä±êעΪ Spring ¹ÜÀíµÄ×é¼þ -public class KafkaUtils { - - private final KafkaTemplate kafkaTemplate; - private final ExecutorService executorService; // ÓÃÓÚÒì²½Ö´ÐÐ - - /** - * ¹¹Ôì·½·¨ - * Spring »á×Ô¶¯×¢Èë KafkaTemplate - * - * @param kafkaTemplate KafkaTemplate ÓÃÓÚ·¢ËÍÏûÏ¢ - */ - public KafkaUtils(KafkaTemplate kafkaTemplate) { - this.kafkaTemplate = kafkaTemplate; - // ³õʼ»¯Ò»¸öÏ̳߳أ¬Ê¹Óà 10 ¸öÏß³Ì - this.executorService = Executors.newFixedThreadPool(10); - } - - /** - * Òì²½·¢ËÍÏûÏ¢µ½Ö¸¶¨µÄ Kafka Ö÷Ìâ - * - * @param topic Kafka Ö÷Ìâ - * @param data ÏûÏ¢ÄÚÈÝ - */ - public void sendMessage(final String topic, final String data) { - executorService.submit(new Callable() { - @Override - public Void call() throws Exception { - try { - SendResult sendResult = kafkaTemplate.send(topic, data).get(); // ×èÈûÖ±µ½·¢ËÍÍê³É - log.info("Message sent successfully: {}", sendResult); - } catch (Exception ex) { - log.error("Kafka send message error = {}, topic = {}", ex.getMessage(), topic); - log.error("Data: {}", data); - } - return null; - } - }); - } - - /** - * ¹Ø±ÕÏß³Ì³Ø - * ÔÚ³ÌÐò½áÊøÊ±ÐèÒªµ÷Óø÷½·¨ÒÔÊÍ·Å×ÊÔ´ - */ - public void shutdown() { - executorService.shutdown(); - } -} diff --git a/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties b/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties index bcc1ddcf9..99a3432fe 100644 --- a/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties +++ b/projects/fangjia-bkoff/src/main/resources/fangjia-bkoff.properties @@ -77,30 +77,4 @@ merchantno=10000469946 #appKey appKey=SQKK10000469946 #\u7EDF\u4E00\u9274\u6743\u7ED1\u5361\u8BF7\u6C42 -authbindcardreqUri=/rest/v1.0/paperorder/unified/pay - -# Kafka bootstrap servers -kafka.bootstrap-servers=192.168.0.89:19092,192.168.0.90:19092,192.168.0.91:19092 - -# Producer configuration -kafka.producer.retries=0 -kafka.producer.batch-size=16384 -kafka.producer.buffer-memory=254288 -kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer -kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer -kafka.producer.auto-topic-creation.auto-create=true -kafka.producer.properties.linger.ms=1 -kafka.producer.properties.session.timeout.ms=15000 -kafka.producer.properties.sasl.mechanism=PLAIN -kafka.producer.properties.security.protocol=SASL_PLAINTEXT -kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456"; - -# Consumer configuration -kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer -kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer -kafka.consumer.group-id=log-data-group -kafka.consumer.auto-offset-reset=earliest -kafka.consumer.properties.session.timeout.ms=15000 -kafka.consumer.properties.sasl.mechanism=PLAIN -kafka.consumer.properties.security.protocol=SASL_PLAINTEXT -kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="Goojia123456"; +authbindcardreqUri=/rest/v1.0/paperorder/unified/pay \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ed3314d22..9db36e3d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4215,7 +4215,6 @@ class CommonDatautils { ElasticUtils.executeSqlQuery(sql, { map-> // if(map.address && map.address.contains("Îå½Ç³¡")){ index++ - println index rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 15b8005d6..822eee441 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{ def cityList = CommonDatautils.findFullNameCitys().reverse() for(def cityObj : cityList){ -// if(cityObj.city == "ÉϺ£ÊÐ"){ +// if(cityObj.city == "ºúÑîºÓÊÐ"){ def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) - println cityObj.city+" "+results.size() - def rows = running(cityObj.city, results) - save(rows) + println cityObj.province+" " +cityObj.city+" "+results.size() + if(results){ + def rows = running(cityObj.city, results) + save(rows) + } // } } } @@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{ }); } latch.await() - return rets; + return rets } def getStandardRegion(mzbRegionMap, district){ @@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{ } saveObj.name = map.name + if(map.cat){ - def arrs = map.cat.split(";|:") - if(arrs.size()>=1){ - saveObj.type = arrs[0] - } - if(arrs.size()>=2){ - saveObj.secondary_type = arrs[1] - }else{ - saveObj.secondary_type = saveObj.type - } - if(arrs.size()>=3){ - def smallstr = arrs[2] - def smallArr = null - if(smallstr.contains("|")){ - smallArr = smallstr.split("|") - } - if(smallArr && smallArr.size()>=1){ - saveObj.small_type = smallArr[0] - }else{ - saveObj.small_type = smallstr - } - }else{ - saveObj.small_type = saveObj.secondary_type - } - - saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type - + saveObj = catWash(saveObj, map) } def location = [:] location.lon = DataHandleUtils.getDoubleRound(6, map.lng) @@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{ def towns = townMap.get(saveObj.district) if(location){ - if(towns){ - for(def town : towns){ - if(town.shape && town.shape.coordinates){ - def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) - if(flag){ - saveObj.township = town.street - break - } + saveObj = addTown(saveObj, towns, location) + } + + if(map.address){ + saveObj = addressWash(saveObj, map, mzbRegionMap, towns) + } + return saveObj + } + + def catWash(saveObj, map){ + def arrs = map.cat.split(";|:") + if(arrs.size()>=1){ + saveObj.type = arrs[0] + } + if(arrs.size()>=2){ + saveObj.secondary_type = arrs[1] + }else{ + saveObj.secondary_type = saveObj.type + } + if(arrs.size()>=3){ + def smallstr = arrs[2] + def smallArr = null + if(smallstr.contains("|")){ + smallArr = smallstr.split("|") + } + if(smallArr && smallArr.size()>=1){ + saveObj.small_type = smallArr[0] + }else{ + saveObj.small_type = smallstr + } + }else{ + saveObj.small_type = saveObj.secondary_type + } + + saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type + return saveObj + } + + def addTown(saveObj, towns, location){ + if(towns){ + for(def town : towns){ + if(town.shape && town.shape.coordinates){ + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + saveObj.township = town.street + break } } + } // if(map.address && !saveObj.township){ // for(def town : towns){ // if(map.address.contains(town.street)){ @@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{ // println map.address +" º¬½ÖµÀ,Õò£¬Ïç" // } // } - } } + return saveObj + } - if(map.address){ - if(map.address.length()<3){ - saveObj.address = "" - }else{ - def address = map.address - def i = 0 - while(i<5){ - if(address && address.toString().startsWith(map.province)){ - address = address.toString().substring(map.province.toString().length()) - } - if(address && address.toString().startsWith(map.city)){ - address = address.toString().substring(map.city.toString().length()) - } + def addressWash(saveObj, map, mzbRegionMap, towns){ + if(map.address.length()<3){ + saveObj.address = "" + }else{ + def address = map.address + def i = 0 + while(i<5){ + if(address && address.toString().startsWith(map.province)){ + address = address.toString().substring(map.province.toString().length()) + } + if(address && address.toString().startsWith(map.city)){ + address = address.toString().substring(map.city.toString().length()) + } - if(address && saveObj.district){ - def tjjObj = mzbRegionMap.get(saveObj.district) - if(tjjObj){ - def alias = tjjObj.alias - if(!alias){ - alias = [] + if(address && saveObj.district){ + def tjjObj = mzbRegionMap.get(saveObj.district) + if(tjjObj){ + def alias = null + if(tjjObj.alias){ + alias = new CopyOnWriteArrayList<>(tjjObj.alias); + }else{ + alias = new CopyOnWriteArrayList<>(); + } + + if(!alias.contains(saveObj.district)){ + alias << saveObj.district + } + if(alias.size()>=2){ + alias = alias.sort(){ a, b -> + b.length().compareTo(a.length()); } - if(!alias.contains(saveObj.district)){ - alias << saveObj.district - } -// if(alias.size()>=2){ -// alias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } + } - for(def regionName : alias){ - if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){ - address = address.substring(regionName.toString().length()) - break - } + for(def regionName : alias){ + if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){ + address = address.substring(regionName.toString().length()) + break } } } + } - if(address && address.toString().startsWith("ÊÐÏ½Çø")){ - address = address.toString().substring(3) - } + if(address && address.toString().startsWith("ÊÐÏ½Çø")){ + address = address.toString().substring(3) + } - if(address && towns){ - for(def town : towns){ - def townAlias = town.alias - if(!townAlias){ - townAlias = [] + if(address && towns){ + for(def town : towns){ + def townAlias = null + if(town.alias){ + townAlias = new CopyOnWriteArrayList<>(town.alias); + }else{ + townAlias = new CopyOnWriteArrayList<>(); + } + + if(!townAlias.contains(town.street)){ + townAlias << town.street + } + if(townAlias.size()>=2){ + townAlias = townAlias.sort(){ a, b -> + b.length().compareTo(a.length()); + } } - if(!townAlias.contains(town.street)){ - townAlias << town.street - } -// if(townAlias.size()>=2){ -// townAlias = townAlias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } - for(def townName : townAlias){ - if(address.startsWith(townName)){ - address = address.substring(townName.toString().length()) + for(def townName : townAlias){ + if(address.startsWith(townName)){ + address = address.substring(townName.toString().length()) // if(!saveObj.township){ // saveObj.township = town.street // } - break - } + break } } } - i++ - } - if(address && address.startsWith("(") && address.endsWith(")")){ - saveObj.address = address.substring(1, address.length()-1) - }else{ - saveObj.address = address } + i++ + } + if(address && address.startsWith("(") && address.endsWith(")")){ + saveObj.address = address.substring(1, address.length()-1) + }else{ + saveObj.address = address } } return saveObj @@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def save(rows){ BulkRequest request = new BulkRequest(); - int batchSize = 1000; + int batchSize = 2000; int processedCount = 0; def client = ElasticUtils89.getClient("192.168.0.89") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy new file mode 100644 index 000000000..c5ea6d76d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy @@ -0,0 +1,54 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils + +@BatchOwner(Watcher.GaoJuan) +class PoiNineSplitBatch extends BaseBatch{ + + static main(args) { + runBatch(new PoiNineSplitBatch(), args) + } + + @Override + void process(Object args) { + run() + } + def run(){ + def i = 0; + while(i<360) { + def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\new_poi\\new_poi_address_split_${i}.xlsx" + } + + File file = new File(fileName); + if (file.exists()) { + ExcelUtils.multiThreadingModify(fileName, { d -> + if (d['C'] && !d['G']) { + println d['C'] +" "+ d['E'] + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E']) + if (jsonObject) { + if (jsonObject.get("street") && jsonObject.get("street") != "null") { + d['G'] = jsonObject.get("street") + } + if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") { + d['H'] = jsonObject.get("streetNumber") + } + if (jsonObject.get("township") && jsonObject.get("township") != "null") { + d['I'] = jsonObject.get("township") + } + } + } + }) + println fileName +" ¾Å¼¶²ð·ÖÍê³É" + }else{ + println fileName +" Îļþ²»´æÔÚ" + } + i++ + } + } +}