From 02c5f23fbf9ebc0c713f4d46928c8ac89e9cc5ca Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 7 Feb 2025 11:35:31 +0800 Subject: [PATCH 1/7] add --- .../gj/district/UpdateDistrictField.groovy | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy new file mode 100644 index 000000000..710eeacca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy @@ -0,0 +1,413 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据链家数据修改小区库中字段 + * 使用到中间映射表 + */ +@BatchOwner(Watcher.GaoJuan) +class UpdateDistrictField extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"] + + def sources = ["链家", "我爱我家", "安居客"] + + def statMap = [:] + + static main(args) { + runBatch(new UpdateDistrictField(), args) + } + + @Override + void process(Object args) { + def sourceMap = [:] + sources.each {source-> + def valMap = findThirdMatchDist(source) + println source +" "+valMap.size() + if(valMap){ + sourceMap.put(source, valMap) + } + } + def margeResults = megerDatas(sourceMap) +// updateDB(margeResults) + } + + def megerDatas(sourceMap){ + def set = new HashSet() + sources.each {source-> + def valMap = sourceMap.get(source) + set.addAll(valMap.keySet()) + } + def lastValues = [] + set.each {id-> + + def dbObj = CommonDatautils.findDistrictById(id) + + def datas = [] + sources.each {source-> + def valMap = sourceMap.get(source) + def sourceEsObj = findCommunities(valMap.get(id)) + if(sourceEsObj){ + if(source == "链家"){ + sourceEsObj.source = "lianjia" + }else if(source == "我爱我家"){ + sourceEsObj.source = "wiwj" + }else if(source == "安居客"){ + sourceEsObj.source = "anjuke" + } + datas << sourceEsObj + } + } + if(dbObj && datas){ + def newValMap = getNewField2(datas, dbObj) + if(newValMap){ + newValMap.put("id", id) + lastValues << newValMap + } + } + } + + statMap.each {source,fieldMap-> + println source +" "+fieldMap.size() + fieldMap.each{f,v-> + println f+" "+ v + } + println "----------------------------------------" + } + return lastValues + } + + + def margeValue(lianjiaMap, wiwjMap, anjukeMap){ + def lianjisIds = lianjiaMap.keySet() + def wiwjIds = wiwjMap.keySet() + def anjukeIds = anjukeMap.keySet() + def set = new HashSet() + set.addAll(lianjisIds) + set.addAll(wiwjIds) + set.addAll(anjukeIds) + + def lastValues = [] + def lianjiaUpdateCount = 0 + def wiwjUpdateCount = 0 + def anjukeUpdateCount = 0 + def lianjiaCountMap = [:] + def wiwjCountMap = [:] + def anjukeCountMap = [:] + set.each {id-> + BasicDBObject newValMap = new BasicDBObject() + fields.each {field-> + def lianjiaValMap = lianjiaMap.get(id) + def wiwjValMap = wiwjMap.get(id) + def anjukeValMap = anjukeMap.get(id) + if(lianjiaValMap && lianjiaValMap.containsKey(field)){ + newValMap.put(field, lianjiaValMap.get(field)) + lianjiaUpdateCount++ + def count = lianjiaCountMap.get(field) + if(!count){ + lianjiaCountMap.put(field, 1) + }else{ + lianjiaCountMap.put(field, count + 1) + } + }else if(wiwjValMap && wiwjValMap.containsKey(field)){ + newValMap.put(field, wiwjValMap.get(field)) + wiwjUpdateCount++ + def count = wiwjCountMap.get(field) + if(!count){ + wiwjCountMap.put(field, 1) + }else{ + wiwjCountMap.put(field, count + 1) + } + }else if(anjukeValMap && anjukeValMap.containsKey(field)){ + newValMap.put(field, anjukeValMap.get(field)) + anjukeUpdateCount++ + def count = anjukeCountMap.get(field) + if(!count){ + anjukeCountMap.put(field, 1) + }else{ + anjukeCountMap.put(field, count + 1) + } + } + } + newValMap.put("id", id) + lastValues << newValMap + } + + lianjiaCountMap.each {k,v-> + println "[链家] "+k+" "+v + } + + wiwjCountMap.each {k,v-> + println "[我爱我家] "+k+" "+v + } + + anjukeCountMap.each {k,v-> + println "[安居客] "+k+" "+v + } + return lastValues + } + + def updateDB(margeResults){ + for(def mr : margeResults){ + def id = mr.get("id") + mr.remove("id") + mr.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr)) + } + } + + def batchUpdateDist(source){ +// def rets = findThirdMatchDist(source) + println "[${source}] 符保条件小区量:"+rets.size() + def updateIndex = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + def allUpdateList = new CopyOnWriteArrayList() + + def sourceMap = new ConcurrentHashMap() + + for(def i=0;i + def count = countMap.get(k) + if(!count){ + countMap.put(k, 1) + }else{ + countMap.put(k, count+1) + } + } + } + + countMap.each {k,v-> + println k +" 更新量:"+v + } + + println "---------------------------------------------------------" + return sourceMap + } + + def running(url, old_match_id){ + def esObj = findCommunities(url) + def dbObj = CommonDatautils.findDistrictById(old_match_id) + if(dbObj){ + def newValMap = getNewField(esObj, dbObj) + return newValMap + } + return null + } + + def getNewField(esObj, dbObj){ + BasicDBObject newValMap = new BasicDBObject() + esObj.each{field, esVal-> + def dbVal = dbObj.get(field) + def newVal = null + if(esVal && field=="developer" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + + } + if(esVal && field=="estate_company" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(esVal && field=="household_count" && esVal!=dbVal){ + newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2) + } + + if(esVal && field=="loudong_count" && esVal!=dbVal){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(esVal && field=="estate_charge" && esVal!=dbVal){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toDouble()!=dbVal){ + newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2) + newVal = newVal.toString() + } + } + if(esVal && field=="car_site_count"){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toInteger()!=dbVal){ + newVal = FiledStrand.processCarSite(numVal) + } + } + if(newVal){ + newValMap.put(field, newVal) + } + + } + + return newValMap + } + + def getNewField2(esDatas, dbObj){ + BasicDBObject newValMap = new BasicDBObject() + def lastValueMap = [:] + def haveValueMap = [:] + fields.each{field-> + for(def esData : esDatas){ + def esVal = esData.get(field) + def dbVal = dbObj.get(field) + def newVal = null + if(haveValueMap.containsKey(field)){ + continue + }else{ + if(lastValueMap.containsKey(field)){ + continue + } + if(esVal){ + + if(field=="estate_charge" || field=="car_site_count") { + esVal = FiledStrand.getNumber(esVal) + } + if(dbVal && esVal.toString()==dbVal.toString()){ + haveValueMap.put(field, esVal) + }else{ + if(field=="developer" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(field=="estate_company" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(field=="household_count" && esVal!=dbVal){ + newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2) + } + + if(esVal && field=="loudong_count" && esVal!=dbVal){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(field=="estate_charge" && esVal!=dbVal){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toDouble()!=dbVal){ + newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2) + newVal = newVal.toString() + } + } + if(field=="car_site_count"){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toInteger()!=dbVal){ + newVal = FiledStrand.processCarSite(numVal) + } + } + } + } + if(newVal){ + lastValueMap.put(field, newVal) + + //以下为统计各网站各字段的更新情况 + def fieldStatMap = statMap.get(esData.source) + if(!fieldStatMap){ + fieldStatMap = [:] + statMap.put(esData.source, fieldStatMap) + } + def count = fieldStatMap.get(field) + if(!count){ + fieldStatMap.put(field, 1) + }else{ + fieldStatMap.put(field, count + 1) + } + } + } + } + } + return lastValueMap + } + + def findThirdMatchDist(source){ + def query = new BasicDBObject("source",source).append("old_match_way", 1) + def keys = new BasicDBObject() + keys.put("url", 1) + keys.put("old_match_id", 1) + keys.put("old_match_city", 1) + keys.put("old_match_district", 1) + keys.put("old_match_name", 1) + def idsMap = [:] + def dbcursor = thirdSeaweedMappingColl.find(query, keys) + for(def db : dbcursor){ + if(!db.old_match_id && db.old_match_name){ + def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district") + if(dbo){ + idsMap.put(dbo._id.toString(), db.url) + } + }else{ + idsMap.put(db.old_match_id, db.url) + } + + } + return idsMap + } + + static def findCommunities(url){ + if(!url){ + return null + } + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url)) + def rets = [] + def datas = null + try{ + datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + }catch(Exception e){ + findCommunities(url) + } + def data = [:] + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + data.put("household_count", d.households) + data.put("loudong_count", d.loudong_cnt) + data.put("developer", d.developer) + data.put("estate_company", d.property_company) + data.put("estate_charge", d.property_fee) + data.put("car_site_count", d.parking_cnt) + } + } + return data; + } +} From 4906378a520c39762f8ac96060045de3af1c607b Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 7 Feb 2025 13:45:51 +0800 Subject: [PATCH 2/7] update --- .../com/fangjia/script/gj/FiledStrand.groovy | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index a3c119bec..514dd9631 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -909,14 +909,18 @@ class FiledStrand { } static def getNumber(def input) { - Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?"); - Matcher matcher = pattern.matcher(input); - List list = new ArrayList(); - while (matcher.find()) { - list.add(matcher.group()); - } - if(list.size()>0){ - return list.get(0); + try{ + Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?"); + Matcher matcher = pattern.matcher(input); + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>0){ + return list.get(0); + } + }catch(Exception e){ + println input+" error" } return null; } From eeada6893fd71570b1dc3d47befae4e0e352754e Mon Sep 17 00:00:00 2001 From: majintao Date: Fri, 7 Feb 2025 14:10:18 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E6=9F=A5?= =?UTF-8?q?=E5=91=A8=E8=BE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PoiController.java | 107 +++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java index 1cc608e7e..b02d7ea4b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java @@ -19,6 +19,7 @@ import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.fangjia.CbmRegionService; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; +import org.apache.commons.codec.binary.Base64; import javax.validation.Valid; import java.util.*; @@ -31,6 +32,8 @@ public class PoiController { private AddressParserService addressParserService; private DistrictService districtService; private static final String[] PROPERTY_SEARCH_API_FIELDS = {"address", "city", "region", "name", "addrAlias"}; + private static final String ES_USER = "developer"; + private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3"; public Object getAntiFraudQuestionAction(@Valid AntiFraudQuestionModel model) { RestfulModelAndView mav = new RestfulModelAndView(); @@ -221,7 +224,7 @@ public class PoiController { public Object searchAction(AddressModel model) { RestfulModelAndView mav = new RestfulModelAndView(); - if (StringUtils.isBlank(model.getLocation()) || StringUtils.isBlank(model.getCity())) { + if (StringUtils.isBlank(model.getLocation()) || StringUtils.isBlank(model.getCity())) { mav.setModel(BasicModel.buildBadRequestModel("城市和经纬度为必填项")); return mav; } @@ -268,6 +271,108 @@ public class PoiController { return mav; } + public Object facilitiesAction(AddressModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + String city = model.getCity(); + String lngStr = model.getLng(); + String latStr = model.getLat(); + String numberPattern = "-?\\d+(\\.\\d+)?"; // 匹配整数或浮点数 + if (!lngStr.matches(numberPattern) || !latStr.matches(numberPattern)) { + mav.setModel(BasicModel.buildBadRequestModel("location(经纬度)格式错误")); + return mav; + } + double lng = Double.parseDouble(lngStr); + double lat = Double.parseDouble(latStr); + Map> result = new HashMap>(); + List bankSmallType = Arrays.asList("银行", "中国人民银行", "国家开发银行", "中国进出口银行"); + List busSmallType = Collections.singletonList("公交车站相关"); + List hospitalSmallType = Arrays.asList("综合医院", "三级甲等医院", "专科医院"); + List schoolSmallType = Arrays.asList("学校", "高等院校", "中学", "小学", "幼儿园"); + List subwaySmallType = Collections.singletonList("地铁站"); + List supermarketSmallType = Arrays.asList("商场", "购物中心", "普通商场"); + + result.put("bank", findAroundByLocation(city, lng, lat, bankSmallType, 1000, true)); + result.put("bus", findAroundByLocation(city, lng, lat, busSmallType, 1000, false)); + result.put("hospital", findAroundByLocation(city, lng, lat, hospitalSmallType, 1000, true)); + result.put("school", findAroundByLocation(city, lng, lat, schoolSmallType, 1000, true)); + result.put("subway", findAroundByLocation(city, lng, lat, subwaySmallType, 1000, false)); + result.put("supermarket", findAroundByLocation(city, lng, lat, supermarketSmallType, 1000, true)); + mav.setModel(BasicModel.buildOKModel().setResult(result)); + return mav; + } + + private List findAroundByLocation(String city, double lng, double lat, List smallTypes, int distance, boolean excludeStatus) { + List poiNames = new ArrayList(); + JSONObject reqBody = new JSONObject(); + JSONObject queryJson = new JSONObject(); + List jsonList = new ArrayList(); + List mustNotList = new ArrayList(); + + if (excludeStatus) { + JSONObject statusParam = new JSONObject(); + statusParam.put("status", 1); + JSONObject statusTerm = new JSONObject(); + statusTerm.put("term", statusParam); + mustNotList.add(statusTerm); + } + JSONObject mainTypeParam = new JSONObject(); + mainTypeParam.put("main_type", 0); + JSONObject mainTypeTerm = new JSONObject(); + mainTypeTerm.put("term", mainTypeParam); + mustNotList.add(mainTypeTerm); + + JSONObject cityParam = new JSONObject(); + cityParam.put("city", city); + JSONObject cityTerm = new JSONObject(); + cityTerm.put("term", cityParam); + jsonList.add(cityTerm); + + JSONObject smallTypeParam = new JSONObject(); + smallTypeParam.put("small_type", smallTypes); + JSONObject smallTypeTerm = new JSONObject(); + smallTypeTerm.put("terms", smallTypeParam); + jsonList.add(smallTypeTerm); + + JSONObject geoDistance = new JSONObject(); + geoDistance.put("distance", distance + "m"); + geoDistance.put("location", Arrays.asList(lng, lat)); + JSONObject geoQuery = new JSONObject(); + geoQuery.put("geo_distance", geoDistance); + jsonList.add(geoQuery); + + queryJson.put("must", jsonList); + if (!mustNotList.isEmpty()) { + queryJson.put("must_not", mustNotList); + } + JSONObject boolQuery = new JSONObject(); + boolQuery.put("bool", queryJson); + reqBody.put("query", boolQuery); + + Base64 token = new Base64(); + String authEncoding = token.encodeAsString((ES_USER + ":" + ES_PASSWORD).getBytes()); + HttpRequest request = new HttpRequest(); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic " + authEncoding); + + try { + String response = request.postWithBody("http://192.168.0.89:9200/gaode_map_poi/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8"); + JSONObject json = JSONObject.fromObject(response); + List> hits = (List>) json.getJSONObject("hits").get("hits"); + + for (Map hit : hits) { + Map source = (Map) hit.get("_source"); + if (source.containsKey("name")) { + poiNames.add(source.get("name").toString()); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + return poiNames; + } + public void setAmapPoiService(AmapPoiService amapPoiService) { this.amapPoiService = amapPoiService; } From 4d1cba094f9ef58f6ab8d23bc0ed9bd1c70e8a57 Mon Sep 17 00:00:00 2001 From: majintao Date: Fri, 7 Feb 2025 15:12:37 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E6=9F=A5?= =?UTF-8?q?=E5=91=A8=E8=BE=B9=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PoiController.java | 9 +- .../open/AdministrativeMappingService.java | 19 ++++ .../AdministrativeMappingServiceImpl.java | 97 +++++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingServiceImpl.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java index b02d7ea4b..f2cb3aa94 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PoiController.java @@ -17,6 +17,7 @@ import com.fangjia.service.api.AmapPoiService; import com.fangjia.service.bo.Point; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.fangjia.CbmRegionService; +import com.fangjia.service.open.AdministrativeMappingService; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.commons.codec.binary.Base64; @@ -31,6 +32,8 @@ public class PoiController { private CbmRegionService cbmRegionService; private AddressParserService addressParserService; private DistrictService districtService; + + private AdministrativeMappingService administrativeMappingService; private static final String[] PROPERTY_SEARCH_API_FIELDS = {"address", "city", "region", "name", "addrAlias"}; private static final String ES_USER = "developer"; private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3"; @@ -273,7 +276,7 @@ public class PoiController { public Object facilitiesAction(AddressModel model) { RestfulModelAndView mav = new RestfulModelAndView(); - String city = model.getCity(); + String city = administrativeMappingService.findCityByCityShort(model.getCity()); String lngStr = model.getLng(); String latStr = model.getLat(); String numberPattern = "-?\\d+(\\.\\d+)?"; // 匹配整数或浮点数 @@ -388,4 +391,8 @@ public class PoiController { public void setDistrictService(DistrictService districtService) { this.districtService = districtService; } + + public void setAdministrativeMappingService(AdministrativeMappingService administrativeMappingService) { + this.administrativeMappingService = administrativeMappingService; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingService.java new file mode 100644 index 000000000..a0fc680e1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingService.java @@ -0,0 +1,19 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.dao.AdministrativeMappingDao; +import com.fangjia.dal.mongo.model.AdministrativeMapping; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public interface AdministrativeMappingService { + + + + public String findCityCodeByCityShort(String cityShort); + public String findCityByCityShort(String cityShort) ; + +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingServiceImpl.java new file mode 100644 index 000000000..521b1e5ef --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AdministrativeMappingServiceImpl.java @@ -0,0 +1,97 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.dao.AdministrativeMappingDao; +import com.fangjia.dal.mongo.model.AdministrativeMapping; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class AdministrativeMappingServiceImpl implements AdministrativeMappingService { + private AdministrativeMappingDao administrativeMappingDao; + private List CITY_SHORT = new ArrayList(Arrays.asList( + "五家渠", "图木舒克", "石河子", "吐鲁番", "中卫", "石嘴山", "海北州", + "张掖", "天水", "铜川", "山南", "昌都", "拉萨", "咸阳", "迪庆州", + "楚雄", "白银", "榆林", "普洱", "丽江", "昆明", "玉树州", "甘孜州", + "阿坝州", "银川", "雅安", "文山州", "西双版纳州", "琼中", "保亭", "乐东", + "昌江", "白沙", "临高", "陇南", "定安", "万宁", "宝鸡", "三亚", "来宾", + "贺州", "怒江州", "河池", "贵港", "防城港", "梧州", "柳州", "中山", "清远", + "定西", "临沧", "德宏州", "琼海", "崇左", "鹤岗", "安阳", "宿迁", + "泰州", "盐城", "齐齐哈尔", "淮安", "澄迈", "营口", "宿州", "甘南", "泰安", + "无锡", "鄂州", "吴忠", "玉林", "鄂尔多斯", "湖州", "辽源", "邯郸", "六安", + "海口", "双鸭山", "抚顺", "白城", "贵阳", "运城", "徐州", "蚌埠", + "白山", "延安", "通化", "牡丹江", "临汾", "云浮", "朝阳", "保山", + "铁岭", "淮南", "金华", "长治", "克孜勒苏", "四平", "太原", "梅州", + "北京", "阜新", "晋中", "阿拉善盟", "晋城", "石家庄", "五指山", "鸡西", + "抚州", "果洛州", "驻马店", "芜湖", "呼和浩特", "邢台", "黑河", "七台河", + "承德", "龙岩", "镇江", "苏州", "衡水", "安康", "南宁", "包头", + "赤峰", "济源", "绍兴", "张家口", "大同", "阳泉", "吕梁", "乌海", + "河源", "通辽", "滁州", "东莞", "杭州", "巴彦淖尔", "常州", "乌兰察布", + "新余", "延边", "九江", "临夏", "湘西", "保定", "林芝", "沧州", + "阜阳", "临沂", "乌鲁木齐", "锡林郭勒盟", "本溪", "德州", "扬州", + "鞍山", "松原", "丽水", "衢州", "合肥", "屯昌", "黄山", "武威", + "亳州", "宣城", "三明", "绥化", "湘潭", "衡阳", "南平", "沈阳", + "辽阳", "南昌", "景德镇", "鹰潭", "赣州", "兴安盟", "吉安", + "庆阳", "宜春", "上饶", "潍坊", "济南", "淄博", "哈尔滨", + "枣庄", "忻州", "济宁", "日照", "韶关", "开封", "商丘", "大理", + "菏泽", "西宁", "郑州", "神农架", "平顶山", "朔州", "鹤壁", + "新乡", "东方", "焦作", "伊春", "萍乡", "常德", "周口", + "渭南", "张家界", "阿里", "桂林", "大庆", "长沙", "益阳", + "株洲", "邵阳", "郴州", "永州", "娄底", "胡杨河", "昆玉", + "嘉峪关", "北屯", "塔城", "巴音郭楞", "博尔塔拉", "昌吉", + "阿拉尔", "克拉玛依", "海西州", "黄南州", "酒泉", "平凉", + "汉中", "昭通", "黔西南", "铜仁", "遵义", "资阳", "巴中", + "广安", "宜宾", "眉山", "南充", "毕节", "乐山", "达州", + "遂宁", "广元", "绵阳", "德阳", "伊犁", "成都", "安顺", + "重庆", "陵水", "红河", "可克达拉", "文昌", "儋州", "百色", + "北海", "揭阳", "潮州", "阳江", "汕尾", "双河", "肇庆", + "湛江", "汕头", "深圳", "内江", "广州", "台州", "嘉兴", + "茂名", "宁波", "三沙", "南通", "和田", "舟山", "连云港", + "喀什", "南京", "锦州", "长春", "固原", "温州", "上海", + "佛山", "钦州", "六盘水", "潜江", "葫芦岛", "岳阳", "曲靖", + "佳木斯", "大兴安岭", "濮阳", "信阳", "烟台", "安庆", + "秦皇岛", "天津", "廊坊", "黔南", "那曲", "海南州", + "日喀则", "武汉", "怀化", "三门峡", "海东", "兰州", + "聊城", "凉山州", "泸州", "南阳", "黔东南", "吉林", + "阿克苏", "滨州", "唐山", "铜陵", "金昌", "丹东", + "玉溪", "白杨", "呼伦贝尔", "铁门关", "仙桃", "随州", + "淮北", "商洛", "江门", "池州", "盘锦", "厦门", + "西安", "自贡", "莆田", "泉州", "漳州", "福州", + "马鞍山", "宁德", "大连", "孝感", "十堰", "青岛", + "攀枝花", "东营", "威海", "新星", "洛阳", "黄石", + "阿勒泰", "许昌", "漯河", "宜昌", "哈密", "襄阳", + "荆门", "荆州", "黄冈", "恩施", "天门", "咸宁", + "惠州", "珠海" + )); + + + public String findCityCodeByCityShort(String cityShort) { + ArrayList params = new ArrayList(); + params.add(new QueryParam("cityShort", cityShort)); + params.add(new QueryParam("type", "城市")); + AdministrativeMapping firstByParams = administrativeMappingDao.findFirstByParams(params); + if (firstByParams != null && StringUtils.isNotBlank(firstByParams.getMzbCityCode())) { + return firstByParams.getMzbCityCode(); + } + return null; + } + + public String findCityByCityShort(String cityShort) { + if (CITY_SHORT.contains(cityShort)) { + ArrayList params = new ArrayList(); + params.add(new QueryParam("cityShort", cityShort)); + params.add(new QueryParam("type", "城市")); + AdministrativeMapping firstByParams = administrativeMappingDao.findFirstByParams(params); + if (firstByParams != null && StringUtils.isNotBlank(firstByParams.getMzbCityCode())) { + return firstByParams.getCity(); + } + } + return cityShort; + } + + public void setAdministrativeMappingDao(AdministrativeMappingDao administrativeMappingDao) { + this.administrativeMappingDao = administrativeMappingDao; + } +} From fbe0709780a00d4b4fcfcf0132b67750fa9de318 Mon Sep 17 00:00:00 2001 From: majintao Date: Fri, 7 Feb 2025 15:48:28 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=88=BF=E6=BA=90=E8=B7=9D=E7=A6=BB?= =?UTF-8?q?=E6=94=B9=E6=88=90=E5=8F=82=E6=95=B0=EF=BC=8C=E9=BB=98=E8=AE=A4?= =?UTF-8?q?3km?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/HouseController.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index 8870454dc..f8bfdcf10 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -927,7 +927,9 @@ public class HouseController { model.setStartTime(sixMonthsAgoInt); model.setEndTime(todayInt); // 默认3000M内 - model.setRadius(3000); + if (model.getRadius() == null) { + model.setRadius(3000); + } if (StringUtils.isBlank(model.getLocation())) { mav.setModel(BasicModel.buildBadRequestModel("location(经纬度)不能为空")); return mav; From c717f3cf6ce6d64285d4ec43ceb1e6a1b202a007 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 7 Feb 2025 16:58:05 +0800 Subject: [PATCH 6/7] update --- .../gj/district/UpdateDistrictField.groovy | 241 +++++++++--------- .../gj/images/AddDistImagesBatch.groovy | 49 ++-- .../thirdPriceCrawl/DataStandardBatch.groovy | 1 + 3 files changed, 150 insertions(+), 141 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy index 710eeacca..f44d8bf4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy @@ -52,7 +52,8 @@ class UpdateDistrictField extends BaseBatch{ } } def margeResults = megerDatas(sourceMap) -// updateDB(margeResults) + updateDB(margeResults) + println "修改完成 "+new Date().toString() } def megerDatas(sourceMap){ @@ -101,74 +102,74 @@ class UpdateDistrictField extends BaseBatch{ } - def margeValue(lianjiaMap, wiwjMap, anjukeMap){ - def lianjisIds = lianjiaMap.keySet() - def wiwjIds = wiwjMap.keySet() - def anjukeIds = anjukeMap.keySet() - def set = new HashSet() - set.addAll(lianjisIds) - set.addAll(wiwjIds) - set.addAll(anjukeIds) - - def lastValues = [] - def lianjiaUpdateCount = 0 - def wiwjUpdateCount = 0 - def anjukeUpdateCount = 0 - def lianjiaCountMap = [:] - def wiwjCountMap = [:] - def anjukeCountMap = [:] - set.each {id-> - BasicDBObject newValMap = new BasicDBObject() - fields.each {field-> - def lianjiaValMap = lianjiaMap.get(id) - def wiwjValMap = wiwjMap.get(id) - def anjukeValMap = anjukeMap.get(id) - if(lianjiaValMap && lianjiaValMap.containsKey(field)){ - newValMap.put(field, lianjiaValMap.get(field)) - lianjiaUpdateCount++ - def count = lianjiaCountMap.get(field) - if(!count){ - lianjiaCountMap.put(field, 1) - }else{ - lianjiaCountMap.put(field, count + 1) - } - }else if(wiwjValMap && wiwjValMap.containsKey(field)){ - newValMap.put(field, wiwjValMap.get(field)) - wiwjUpdateCount++ - def count = wiwjCountMap.get(field) - if(!count){ - wiwjCountMap.put(field, 1) - }else{ - wiwjCountMap.put(field, count + 1) - } - }else if(anjukeValMap && anjukeValMap.containsKey(field)){ - newValMap.put(field, anjukeValMap.get(field)) - anjukeUpdateCount++ - def count = anjukeCountMap.get(field) - if(!count){ - anjukeCountMap.put(field, 1) - }else{ - anjukeCountMap.put(field, count + 1) - } - } - } - newValMap.put("id", id) - lastValues << newValMap - } - - lianjiaCountMap.each {k,v-> - println "[链家] "+k+" "+v - } - - wiwjCountMap.each {k,v-> - println "[我爱我家] "+k+" "+v - } - - anjukeCountMap.each {k,v-> - println "[安居客] "+k+" "+v - } - return lastValues - } +// def margeValue(lianjiaMap, wiwjMap, anjukeMap){ +// def lianjisIds = lianjiaMap.keySet() +// def wiwjIds = wiwjMap.keySet() +// def anjukeIds = anjukeMap.keySet() +// def set = new HashSet() +// set.addAll(lianjisIds) +// set.addAll(wiwjIds) +// set.addAll(anjukeIds) +// +// def lastValues = [] +// def lianjiaUpdateCount = 0 +// def wiwjUpdateCount = 0 +// def anjukeUpdateCount = 0 +// def lianjiaCountMap = [:] +// def wiwjCountMap = [:] +// def anjukeCountMap = [:] +// set.each {id-> +// BasicDBObject newValMap = new BasicDBObject() +// fields.each {field-> +// def lianjiaValMap = lianjiaMap.get(id) +// def wiwjValMap = wiwjMap.get(id) +// def anjukeValMap = anjukeMap.get(id) +// if(lianjiaValMap && lianjiaValMap.containsKey(field)){ +// newValMap.put(field, lianjiaValMap.get(field)) +// lianjiaUpdateCount++ +// def count = lianjiaCountMap.get(field) +// if(!count){ +// lianjiaCountMap.put(field, 1) +// }else{ +// lianjiaCountMap.put(field, count + 1) +// } +// }else if(wiwjValMap && wiwjValMap.containsKey(field)){ +// newValMap.put(field, wiwjValMap.get(field)) +// wiwjUpdateCount++ +// def count = wiwjCountMap.get(field) +// if(!count){ +// wiwjCountMap.put(field, 1) +// }else{ +// wiwjCountMap.put(field, count + 1) +// } +// }else if(anjukeValMap && anjukeValMap.containsKey(field)){ +// newValMap.put(field, anjukeValMap.get(field)) +// anjukeUpdateCount++ +// def count = anjukeCountMap.get(field) +// if(!count){ +// anjukeCountMap.put(field, 1) +// }else{ +// anjukeCountMap.put(field, count + 1) +// } +// } +// } +// newValMap.put("id", id) +// lastValues << newValMap +// } +// +// lianjiaCountMap.each {k,v-> +// println "[链家] "+k+" "+v +// } +// +// wiwjCountMap.each {k,v-> +// println "[我爱我家] "+k+" "+v +// } +// +// anjukeCountMap.each {k,v-> +// println "[安居客] "+k+" "+v +// } +// return lastValues +// } def updateDB(margeResults){ for(def mr : margeResults){ @@ -179,57 +180,57 @@ class UpdateDistrictField extends BaseBatch{ } } - def batchUpdateDist(source){ -// def rets = findThirdMatchDist(source) - println "[${source}] 符保条件小区量:"+rets.size() - def updateIndex = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(rets.size()) - def allUpdateList = new CopyOnWriteArrayList() - - def sourceMap = new ConcurrentHashMap() - - for(def i=0;i - def count = countMap.get(k) - if(!count){ - countMap.put(k, 1) - }else{ - countMap.put(k, count+1) - } - } - } - - countMap.each {k,v-> - println k +" 更新量:"+v - } - - println "---------------------------------------------------------" - return sourceMap - } +// def batchUpdateDist(source){ +//// def rets = findThirdMatchDist(source) +// println "[${source}] 符保条件小区量:"+rets.size() +// def updateIndex = new AtomicLong() +// final CountDownLatch latch = new CountDownLatch(rets.size()) +// def allUpdateList = new CopyOnWriteArrayList() +// +// def sourceMap = new ConcurrentHashMap() +// +// for(def i=0;i +// def count = countMap.get(k) +// if(!count){ +// countMap.put(k, 1) +// }else{ +// countMap.put(k, count+1) +// } +// } +// } +// +// countMap.each {k,v-> +// println k +" 更新量:"+v +// } +// +// println "---------------------------------------------------------" +// return sourceMap +// } def running(url, old_match_id){ def esObj = findCommunities(url) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy index 60824f8d5..4c710cb47 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy @@ -38,7 +38,7 @@ class AddDistImagesBatch extends BaseBatch{ @Override public void process(Object args) { - def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") + def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) // def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) def citys = CommonUtils.findAllCity() citys.each {city-> @@ -48,7 +48,7 @@ class AddDistImagesBatch extends BaseBatch{ matchCommunitys(communitys) } -// statImgCount() + statImgCount() } def findAllEsData2(city, mdate){ QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true) @@ -105,40 +105,47 @@ class AddDistImagesBatch extends BaseBatch{ def total = communitys.size() for(def i=0;i Date: Mon, 10 Feb 2025 14:57:57 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E8=A1=A5=E5=85=A8=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E5=8A=A0=E5=9C=B0=E5=9D=80=E5=88=AB=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/DistrictController.java | 10 +++++++++- .../java/com/fangjia/service/api/AutoCompleteBo.java | 10 ++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java index 442e4b77a..17ab2c4b2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java @@ -146,12 +146,20 @@ public class DistrictController { if (!district.getName().contains(keyword)) { if (district.getAlias() != null && !district.getAlias().isEmpty()) { for (String alias : district.getAlias()) { - if (alias.contains(keyword)){ + if (alias.contains(keyword)) { bo.setAlias(alias); break; } } } + if (district.getAddrAlias() != null && !district.getAddrAlias().isEmpty()) { + for (String addrAlias : district.getAddrAlias()) { + if (addrAlias.contains(keyword)) { + bo.setAddrAlias(addrAlias); + break; + } + } + } } ret.add(bo); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AutoCompleteBo.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AutoCompleteBo.java index 9e714a4bb..d21cfa34c 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AutoCompleteBo.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AutoCompleteBo.java @@ -24,6 +24,8 @@ public class AutoCompleteBo { private String alias; + private String addrAlias; + /*** @Deprecated end ****/ public String getName() { @@ -145,4 +147,12 @@ public class AutoCompleteBo { public void setAlias(String alias) { this.alias = alias; } + + public String getAddrAlias() { + return addrAlias; + } + + public void setAddrAlias(String addrAlias) { + this.addrAlias = addrAlias; + } }