diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java new file mode 100644 index 000000000..004f28bb8 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDao.java @@ -0,0 +1,16 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public interface AliasFangDao extends MongoDao { + + List findFangByNameAndAlias(String city, String region, Set names); +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java new file mode 100644 index 000000000..e5fe59202 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/AliasFangDaoImpl.java @@ -0,0 +1,40 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.model.AliasFang; +import com.fangjia.dal.mongo.model.ConsumerAddress; +import com.mongodb.BasicDBList; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import org.apache.commons.lang.StringUtils; +import org.sothis.core.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public class AliasFangDaoImpl extends MongoDaoImpl implements AliasFangDao{ + + @Override + public List findFangByNameAndAlias(String city, String region, Set names) { + DBObject query = new BasicDBObject(); + if (StringUtils.isNotBlank(city)) { + query.put("city", city); + } + if (StringUtils.isNotBlank(region)) { + query.put("region", region); + } + // db.alias_fang.find({'city': '重庆', 'region': '渝北', $or: [{'name': '中国摩'}, {'alias': '中国摩'}]}) + if(CollectionUtils.isNotEmpty(names)) { + BasicDBList dbList = new BasicDBList(); + dbList.add(new BasicDBObject("name", new BasicDBObject("$in", names))); + dbList.add(new BasicDBObject("alias", new BasicDBObject("$in", names))); + query.put("$or", dbList); + } + List objs = find(query, null, null, 0, Integer.MAX_VALUE); + return objs; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java new file mode 100644 index 000000000..c4027ecc3 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AliasFang.java @@ -0,0 +1,68 @@ +package com.fangjia.dal.mongo.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; + +/** + * 搜房名称与别名关系 + */ +@Entity +@Table(name = "business.alias_fang") +public class AliasFang implements MongoEntity { + + private String id; + private String city; + private String region; + private String name; + private String alias; + + @Override + @Id + @GeneratedValue + @Column(name = "_id", unique = true, nullable = false) + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "alias") + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 2bb00ca36..00d35fa46 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -187,7 +187,7 @@ public class District implements SeaweedEntity { private double score; // 小区性质 private String landProperty; - private String masterDistrictNname; + private String masterDistrictName; private String ip; private String addrType; private boolean selected; @@ -309,12 +309,12 @@ public class District implements SeaweedEntity { } @Column(name = "master_district_name") - public String getMasterDistrictNname() { - return masterDistrictNname; + public String getMasterDistrictName() { + return masterDistrictName; } - public void setMasterDistrictNname(String masterDistrictNname) { - this.masterDistrictNname = masterDistrictNname; + public void setMasterDistrictName(String masterDistrictName) { + this.masterDistrictName = masterDistrictName; } public void setCategory(String category) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index c0f3d23ac..5f9b37bb7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -391,13 +391,106 @@ public class AddrSvcController { } }; - /** * 涉农识别-新规则-南京银行 * * @param model * @return */ + public Object njcbVillageIdentifyAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); + String address = model.getAddress(); + params.put("address", address); + String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null; + params.put("type", type); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8"); + } catch (Exception e) { + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("msgCode"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if (json.containsKey("result")) { + //我们的结果 + JSONObject result = json.getJSONObject("result"); + //我们省级及以上有开发区,直接用我们的结果 + if (result.containsKey("devDistrictName")) { + basicModel.setResult(result); + + } else { + try { + //如果是江苏省或者浙江省 调用齐梁的结果 + if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) { + //结果比较,如果涉农结果不一致 + //调用齐梁 type :1 对公 0 对私 + //我们 type :1 对私 2 对共 + String qltype = "2".equals(type) ? "1" : "0"; + Map map = senRequest(address, qltype); + // 涉农结果不一样 + if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) { + //城市不一样 用我们 + if (result.containsKey("city")) { + if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) { + basicModel.setResult(result); + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + + } else { + basicModel.setResult(result); + } + + } else { + basicModel.setResult(result); + + } + + } catch (Exception e) { + e.printStackTrace(); + basicModel.setResult(result); + } + + } + + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + + /** + * 涉农识别-新规则-基础版本 + * + * @param model + * @return + */ public Object villageIdentifyAction(AddressModel model) { RestfulModelAndView mav = new RestfulModelAndView(); @@ -435,11 +528,8 @@ public class AddrSvcController { if (json.containsKey("result")) { //我们的结果 JSONObject result = json.getJSONObject("result"); - // 如果是无效信息,则用我们的 - if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) { - basicModel.setResult(result); - }//我们省级及以上有开发区,直接用我们的结果 - else if (result.containsKey("devDistrictName")) { + //我们省级及以上有开发区,直接用我们的结果 + if (result.containsKey("devDistrictName")) { basicModel.setResult(result); } else { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java index 4d264b835..9d891b609 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java @@ -233,10 +233,13 @@ public class ComprehensiveQueryController { communityAvgPriceParams.put("city", model.getCity()); communityAvgPriceParams.put("address", model.getAddress()); communityAvgPriceParams.put("category", model.getCategory()); - communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); +// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); + communityAvgPriceParams.put("suggestName", String.valueOf(model.getName())); + communityAvgPriceParams.put("preciseLevel", "strict"); + communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"); JSONObject matchResponse = null; try { - matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8"); + matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8"); // matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8"); } catch (HttpRequestException e) { @@ -256,9 +259,21 @@ public class ComprehensiveQueryController { DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district)); if (null != statBo) { if (statBo.getDistrictStat() != null) { - dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); + // 有2B价格,使用2B价格 + Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b(); + if (avgPrice2b != null && avgPrice2b > 0) { + dto.setAvgPrice(avgPrice2b); + } else { + dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); + } } else if (statBo.getVillaDistrictStat() != null) { - dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); + // 有2B价格,使用2B价格 + Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b(); + if (avgPrice2b != null && avgPrice2b > 0) { + dto.setAvgPrice(avgPrice2b); + } else { + dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); + } } } @@ -638,10 +653,18 @@ public class ComprehensiveQueryController { if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { int factorPrice = evaluation.getAvgPrice(); int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); - Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95); - Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05); + // 有2B价格用2B价格,没有用2C + Integer minPrice = 0; + Integer maxPrice = 0; + if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) { + minPrice = (int) (evaluation.getAvgPrice2b() * 0.95); + maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05); + } else { + minPrice = (int) (evaluation.getAvgPrice() * 0.95); + maxPrice = (int) (evaluation.getAvgPrice() * 1.05); + } // 偏离度超过5%不进行调整 - if (minPrice <= avgPrice && avgPrice <= maxPrice) { + if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) { evaluation.setAvgPrice(avgPrice); evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); } @@ -657,7 +680,11 @@ public class ComprehensiveQueryController { districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); - districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); + if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) { + districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b()); + } else { + districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); + } // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -882,7 +909,11 @@ public class ComprehensiveQueryController { otherPriceRets = new HashMap(); } - otherPriceRets.put("fjw_price", evaluation.getAvgPrice()); + if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){ + otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b()); + }else { + otherPriceRets.put("fjw_price", evaluation.getAvgPrice()); + } Map levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit()); Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java index 06d3ef504..fd1709bf3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -51,7 +51,7 @@ public class GdBankController { }; /** - * 涉农识别-新规 + * 涉农识别-新规则-光大银行 * * @param model * @return @@ -95,8 +95,8 @@ public class GdBankController { } else if (("否").equals(result.get("countyLevelCity"))) { result.put("prefectureLevelCity", "是"); } - // 如果是无效信息,则用我们的 - if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) { + // 如果是无效信息或者没有行政区划,则用我们的 + if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) { basicModel.setResult(result); //我们省级及以上有开发区,直接用我们的结果 } else if (result.containsKey("devDistrictName")) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 81a281610..b5efca725 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -10,17 +10,7 @@ import java.net.URLEncoder; import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,6 +20,7 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; +import com.fangjia.dal.mongo.dao.AliasFangDao; import com.fangjia.dal.mongo.model.*; import com.fangjia.dal.mongo.open.model.CommunityBasePrice; import com.fangjia.dal.mongo.open.model.CommunityPoi; @@ -39,6 +30,7 @@ import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.open.dto.addressGovernance.StructureDto; import com.fangjia.open.util.CommonDtoUtils; +import com.fangjia.service.address.AddressParserUtil; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; @@ -138,7 +130,6 @@ import com.mongodb.DBObject; @Authority(role = Role.USER) public class PropertyController { - private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class); private DistrictService districtService; private AddressParser addressParser; @@ -186,7 +177,7 @@ public class PropertyController { private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; - private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" }; + private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" }; static final int BAI_DU_GEO = 1;// 百度系坐标 static final int GAO_DE_GEO = 2;// 高德系坐标 @@ -196,6 +187,7 @@ public class PropertyController { private DistrictRoomSqmetreRentService districtRoomSqmetreRentService; private RentDealPriceService rentDealPriceService; private EvaluateRatioService evaluateRatioService; + private AliasFangService aliasFangService; private static final int EVALUATE_SHOW_DETAIL = 1; private static final Map PROPERTY_TYPES = new HashMap() { @@ -609,6 +601,25 @@ public class PropertyController { dto.setBlock(district.getBlock()); } dto.setPropertyType(district.getEstateType2()); + if (CollectionUtils.isNotEmpty(district.getAlias())) { + // 将最相近的别名输出给用户 + String alias = null; + double similarity = 0; + if (StringUtils.isNotBlank(model.getKeyword())) { + for (String mappingAlias : district.getAlias()) { + double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword()); + if (curr > similarity ) { + similarity = curr; + alias = mappingAlias; + } + } + } + if (StringUtils.isNotBlank(alias)) { + dto.setAlias(alias); + } else { + dto.setAlias(district.getAlias().get(0)); + } + } list.add(dto); } @@ -2516,7 +2527,7 @@ public class PropertyController { model.setMax(100); } - List districtNames = new ArrayList(); + Set districtNames = new HashSet(); if (null == district) { districtNames.add(model.getName()); } else { @@ -2524,14 +2535,56 @@ public class PropertyController { if (district.getAlias() != null) { districtNames.addAll(district.getAlias()); } - } - List dealPrices = null; - for (String name : districtNames) { - dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model); - if (!dealPrices.isEmpty()) { - break; + final String masterDistrictName = district.getMasterDistrictName(); + if (StringUtils.isNotBlank(masterDistrictName)) { + List districts; + if ("master".equals(masterDistrictName)) { + // 所有的分期数据 + districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName()); + } else { + // 根据主小区名查询所有名称 + districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName); + } + if (CollectionUtils.isNotEmpty(districts)) { + for (District districtTemp : districts) { + districtNames.add(districtTemp.getName()); + } + } + + // districtNames 不包含 主小区名,加进去 + if (!districtNames.contains(masterDistrictName)){ + districtNames.add(masterDistrictName); + } + } } + // 将搜房中的别名与名称加入查询集合 + try { + List aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames); + Set aliasSet = new HashSet(); + for (AliasFang aliasFang : aliasFangs) { + if (StringUtils.isNotBlank(aliasFang.getName())) { + aliasSet.add(aliasFang.getName()); + } + if (StringUtils.isNotBlank(aliasFang.getAlias())) { + aliasSet.add(aliasFang.getAlias()); + } + } + districtNames.addAll(aliasSet); + } catch (Exception ex) { + ex.printStackTrace(); + } + + +// List dealPrices = new ArrayList(); + // 将主名与映射别名查得成交加入 存在不能查出最新成交问题 +// for (String name : districtNames) { +// List dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model); +// if (CollectionUtils.isNotEmpty(dealPricesTempList)) { +// dealPrices.addAll(dealPricesTempList); +// } +// } + List dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList(districtNames), model.getMax() * 5, model); if (model.getSize() != null) { Collections.sort(dealPrices, new Comparator() { @Override @@ -2580,15 +2633,9 @@ public class PropertyController { } deal = new LinkedHashMap(); dealPrice.setId(null); - if (district != null) { - deal.put("city", district.getCity()); - deal.put("district", district.getRegion()); - deal.put("name", district.getName()); - } else { - deal.put("city", dealPrice.getCity()); - deal.put("district", dealPrice.getRegion()); - deal.put("name", dealPrice.getDistrictName()); - } + deal.put("city", dealPrice.getCity()); + deal.put("district", dealPrice.getRegion()); + deal.put("name", dealPrice.getDistrictName()); if (dealPrice.getArea() == null) { deal.put("size", ""); } else { @@ -3501,6 +3548,33 @@ public class PropertyController { return null; } + private List findDealPricesByNames(String city, String region, List names, int max, + DealPriceModel model) { + List params = new ArrayList(); + params.add(new QueryParam("city", city)); + params.add(new QueryParam("region", region)); + params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names)); + if (model.getRoom() != null) { + params.add(new QueryParam("room", model.getRoom())); + } + if (model.getHall() != null) { + params.add(new QueryParam("hall", model.getHall())); + } + // 18年之后抓住的数据没有score字段 +// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l)); + List dealPrices = dealPriceService.findDealPrices(params, max); + List distinctDealPrices = new ArrayList(); + for (DealPrice dealPrice : dealPrices) { + if (dealPrice.getAvgPrice() == null) { + dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea())); + } + if (!checkExists(distinctDealPrices, dealPrice)) { + distinctDealPrices.add(dealPrice); + } + } + return distinctDealPrices; + } + private List findDealPricesByName(String city, String region, String name, int max, DealPriceModel model) { List params = new ArrayList(); @@ -5959,4 +6033,8 @@ public class PropertyController { public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { this.communityBasePriceService = communityBasePriceService; } + + public void setAliasFangService(AliasFangService aliasFangService) { + this.aliasFangService = aliasFangService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java index 8e9097e2a..aac0f83b2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java @@ -15,6 +15,10 @@ public class PropertyDto { private String propertyType; private String block; private String distance; + /** + * 新增小区别名 + */ + private String alias; public String getId() { return id; } @@ -88,4 +92,12 @@ public class PropertyDto { public void setDistance(String distance) { this.distance = distance; } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 210e810a2..bcd484c2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql +import org.bson.types.ObjectId import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -61,61 +62,67 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd") + def objectId = dateToObjectId(date) + println objectId - println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ - addDealPriceMatch(city); - } + //664a22000000000000000000 5-20 + //664b73800000000000000000 5-21 + def citys = CcbCommonUtils.getRunningCitys(args) +// +// println "curDate:"+curDate +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } // updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 -// for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) -// println "历史id加载量:"+dealIds.size() -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -//// delPgData() -// saveToPgSql(rows) -// -//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") -//// saveToPgSql2(datas) + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) + println "历史id加载量:"+dealIds.size() + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + +// delPgData() + saveToPgSql(rows) + +// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +// saveToPgSql2(datas) } def readTxt(def fileName){ @@ -682,7 +689,13 @@ class BigdataDealBatch extends BaseBatch{ } def updateCrawlTradeDate(){ - + def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20" + def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd") + def objectId = dateToObjectId(date) + def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2) + def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1) + println(objectId) + CcbCommonUtils.updateTrades(startDate, endDate, objectId) } @@ -822,4 +835,15 @@ class BigdataDealBatch extends BaseBatch{ println "end: "+new Date() } + + + def dateToObjectId(Date date) { + // 获取时间戳(单位:秒) + def timestamp = (date.time / 1000) as int + def hexTimestamp = Integer.toHexString(timestamp) + + // 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值 + def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000' + return new ObjectId(objectIdStr) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 84f840c02..7a4062270 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -15,9 +15,10 @@ import groovy.sql.Sql class CcbCommonUtils { - // - public static def citys_fjw_all = [ "乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] - public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] + + public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + + public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] @@ -439,7 +440,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","新泾三村") +// q.put("name","鸿运大院") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -711,4 +712,16 @@ class CcbCommonUtils { return map } + public static def updateTrades(startDate, endDate, cDate){ + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("_id", new BasicDBObject('$gte', cDate)) + def count = DEAL_PRICE_COLL.count(q) + if(count>0){ + def udata = new BasicDBObject() + udata.put("trade_date", endDate) + DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata)) + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 023cc157d..5072828fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -389,7 +389,7 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" // def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all - def citys = CcbCommonUtils.citys_all + def citys = ["上海","广州"]//CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 9ad3f9a31..c43a791b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index c42ab1349..d527b08dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> if(d['J']){ @@ -1301,20 +1301,44 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['A']){ + if(!d['A'] && d['J'] instanceof String){ def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ d['A'] = dist.city d['B'] = dist.region d['C'] = dist.name - d['K'] = dist.estate_type2 - d['L'] = dist.status - d['M'] = dist.name_cat +// d['K'] = dist.estate_type2 +// d['L'] = dist.status +// d['M'] = dist.name_cat + +// d['H'] = dist.name } } } },0) + +//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" +//ExcelUtils.multiThreadingModify(filePath, { d -> +// if(d['B']){ +// +// def obj = CommonDatautils.findDistrictById(d['B']) +// if(obj){ +// def markMap = obj.mark_map +// if(!obj.mark_map){ +// markMap = [:] +// } +// def data = new BasicDBObject() +// markMap.put("exclusive", "是") +// data.put("mark_map", markMap) +// data.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) +// } +// } +// +//}) + + //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" //ExcelUtils.multiThreadingModify(fileName, { d -> // if(d['B'] && !d['D']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index b0756a439..4e70b43c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch { // } // } // - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 479813af4..535748054 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{ def size = regionDatas.size() final CountDownLatch latch = new CountDownLatch(size) def totalCount = new AtomicLong() + for(def i = 0;i3){ - sell_avg_price = avgValue(sellMap.avg_price) + if(sellMap){ sell_avg_area = avgValue(sellMap.area) - sell_avg_total = avgValue(sellMap.total_price) - }else{ - sell_count = 0 - sellDataType = null + sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price) +// if(sell_count>3){ +// if(allCcbPriceMap.containsKey(dist._id.toString())){ +// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price) +// sell_avg_area = avgValue(sellMap.area) +// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price) +// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString()) +// }else{ +// sell_count = 0 +// sellDataType = null +// } +// }else{ +// sell_count = 0 +// sellDataType = null +// } } } + def rent_avg_price = null def rent_avg_area = null def rent_avg_total = null @@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{ rentDataType = null } } - if(!sell_avg_price){ - sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) - } + + //三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格 +// if(!sell_avg_price){ +// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) +// } if(!rent_avg_price){ rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) } @@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{ def sell_avg_area = null def sell_avg_total = null def sellStartDate = null + + def avgPrice = null + if(region){ + avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6)) + }else{ + avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6)) + } + + if (sell_count < 10) { sellDataType = 3 sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); @@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{ sellStartDate = startDate } if (sell_count) { - sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") + sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area") sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price") } @@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{ return datas; } + def findCommunityPriceByCityMap(cityCode, month){ + def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def dataMap = [:] + pgsqlInstance.eachRow(sql, { r -> + dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc) + }) + return dataMap; + } + + def findCommunityPriceCountByCityMap(cityCode, month){ + def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}"; + def dataMap = [:] + pgsqlInstance.eachRow(sql, { r -> + if(r.gujiabdsrt3mlgcasenum){ + dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum) + } + }) + return dataMap; + } + def findCcbCityPrice(cityCode, month){ def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; def avgPrice = null @@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{ return regionPriceMap; } + def findCcbRegionPriceByRegion(cityCode, regionCode, month){ + def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'"; + def price = null + pgsqlInstance.eachRow(sql, { r -> + price = r.gujia_bds_clc_avprc + }) + return price; + } + def districtCityPrictRate(cityPrice, datas){ def distCityRateMap = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ea417b329..e9fa92613 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -13,11 +13,13 @@ import com.mongodb.AggregationOutput import com.mongodb.client.MongoCursor import com.mongodb.client.model.Accumulators import com.mongodb.client.model.Aggregates +import com.rabbitmq.client.impl.AMQImpl import groovy.json.JsonSlurper import net.sf.json.JSONArray import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils +import org.apache.poi.ss.util.RegionUtil import org.bson.Document import org.bson.types.ObjectId @@ -117,6 +119,7 @@ class CommonDatautils { } public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009") + public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -274,7 +277,7 @@ class CommonDatautils { q.put("status", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","泰瑞家园") +// q.put("name","托斯卡纳庄园") DBObject keys = new BasicDBObject(); if(fields){ @@ -283,7 +286,7 @@ class CommonDatautils { } } - def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) + def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000) List rets = [] for(def obj : cursor){ rets << obj @@ -4571,6 +4574,28 @@ class CommonDatautils { return townMap } + static def findAllTownRanges(){ + def q = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("town", 1) + def dbcursor = TOWN_POLYLINECOLL.find(q, keys) + def townMap = [:] + for(def d : dbcursor){ + def lists = townMap.get(d.city) + if(!lists){ + lists = [] + } + def t = [:] + t.region = d.region + t.town = d.town + lists << t + townMap.put(d.city, lists) + } + return townMap + } + static def queryFullNameCity(){ def q = new BasicDBObject() def dbcursor = administrative_division_100_CONN.find(q) @@ -4605,6 +4630,108 @@ class CommonDatautils { } + static def queryFullNameByCity(){ + def q = new BasicDBObject() + q.put("type","城市") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + def cityMap = [:] + for(def d : dbcursor){ + def city = RegionData.formatCity(d.city) + if(city){ + cityMap.put(city, d) + }else{ + //println d.city +" 格式化不出" + } + + } + return cityMap + } + + static def queryDistNameByCity(){ + + def q = new BasicDBObject() + q.put("type","区域") + def keys = new BasicDBObject() + keys.put("province",1) + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + keys.put("mzb_district_name", 1) + keys.put("shape", 1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000) + def regionMap = [:] + for(def d : dbcursor){ + if(d.mzb_district_name){ + def regions = regionMap.get(d.city) + if(!regions){ + regions = [] + } + regions << d + regionMap.put(d.city, regions) + }else{ + println d.province+" "+d.district +" 格式化不出民政部区域" + } + } + return regionMap + } + + static def queryCommunityByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("province",new BasicDBObject('$in', ['福建省','重庆市'])) + + def keys = new BasicDBObject() + keys.put("province",1) + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + keys.put("shape",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.district + def list = townMap.get(key) + if(!list){ + list = [] + } + list << d + townMap.put(key, list) + } + return townMap + + } + + + static def queryNewTownByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city + def list = townMap.get(key) + if(!list){ + list = [] + } + def t = [:] + t.region = d.district + t.town = d.street + list << t + townMap.put(key, list) + } + return townMap + + } + + static def queryDistNameCity(){ def q = new BasicDBObject() @@ -4675,6 +4802,54 @@ class CommonDatautils { } + static def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index aa7425f23..2003b4c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -24,13 +26,14 @@ class DevProcess { DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") static main(args) { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() - task.devNameSplitRoad() + task.devSplitRoad() +// task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -44,7 +47,15 @@ class DevProcess { // task.addNewDev() // task.createDevMapping() // task.deleteAddr() +// task.deleteAddr2() // task.synchDevTo43() +// task.addNewDev2() + +// task.addAlias() +// task.addDevShape()//添加开发区围栏 +// task.matchDevTownship()//添加乡镇街道 + +// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中 } @@ -162,69 +173,101 @@ class DevProcess { } } + def findDev(city, name){ + def q = new BasicDBObject() + q.put("name", name) + q.put("city", city) + def obj = development_zone_CONN.findOne(q) + return obj + } + def devSplitRoad(){ - def i = 10; - while(i<11){ - def fileName = "/home/goojia/gj/company-${i}.xlsx" + def i = 0; + while(i<20){ + def fileName = "/home/goojia/gj/qzd-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\company-${i}.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx" } - ExcelUtils.multiThreadingModify(fileName,{d-> - /*if(d['C'] && !d['V']){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['V'] = jsonObject.get("province") - } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['W'] = jsonObject.get("city") - } - - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['X'] = jsonObject.get("district") - } - - if(jsonObject && street && streetNumber && street!="null"){ - d['Y'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['Z'] = jsonObject.get("streetNumber") - } - } - }*/ - if(d['C'] && d['B']!='河北省'){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C']) - if(jsonObject){ - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['J'] = jsonObject.get("province") - } - - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['K'] = jsonObject.get("city") - } - - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['L'] = jsonObject.get("district") - } - - if(jsonObject && street && streetNumber && street!="null"){ - d['M'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['N'] = jsonObject.get("streetNumber") + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['E'] && !d['G']){ + def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E']) + if(jsonObject1) { + if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") { + def devName = jsonObject1.get("dev") + def obj = findDev(d['B'], devName) + d['F'] = devName + if (obj) { + d['Q'] = obj._id.toString() + d['R'] = obj.name + } } } - - if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ - d['O'] = jsonObject.get("dev") - } } - } - },0) - i++ + if(d['C'] && !d['S']) { + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if (jsonObject.get("province") && jsonObject.get("province") != "null") { + d['G'] = jsonObject.get("province") + } + + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + d['H'] = jsonObject.get("city") + } + + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + d['I'] = jsonObject.get("district") + } + + if (jsonObject.get("township") && jsonObject.get("township") != "null") { + d['J'] = jsonObject.get("township") + } + + if (jsonObject && street && streetNumber && street != "null") { + d['K'] = jsonObject.get("street") + def road = jsonObject.get("street") + if (streetNumber != 'null') { + d['L'] = jsonObject.get("streetNumber") + road = road + jsonObject.get("streetNumber") + } else { + d['N'] = "信息不完整" + } + + if (!d['N'] && !d['C'].contains(road)) { + d['N'] = "原地址不含路" + } + + if (!d['N']) { + def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$') + if (!flag) { + d['N'] = "路非法" + } + def roadsize = jsonObject.get("street").size() + if (!d['N'] && (roadsize < 3 || roadsize > 7)) { + d['N'] = "路太长|路太短" + } + } + }else{ + d['N'] = "无地址" + } + + if (jsonObject.get("dev") && jsonObject.get("dev") != "null") { + def devName = jsonObject.get("dev") + def obj = findDev(d['B'], devName) + d['M'] = devName + if (obj) { + d['S'] = obj._id.toString() + d['T'] = obj.name + } + } + } + },0) + i++ + } } } @@ -234,7 +277,7 @@ class DevProcess { def devNameSplitRoad(){ def fileName = "/home/goojia/gj/company.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> if(d['P'] && d['P']=="路正常" && !d['O']){ @@ -291,17 +334,18 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> - if(d['H']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d-> + if(!d['Q'] && d['L']){ def q = new BasicDBObject() - q.put("name", d['H']) + q.put("name", d['L']) + q.put("city",d['G']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['S'] = obj._id.toString() - d['T'] = obj.name + d['Q'] = obj._id.toString() + d['R'] = obj.name } } - },1); + },0); } def calAddrDistince(){ @@ -506,6 +550,27 @@ class DevProcess { }) } + def addNewDev2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + if(d['F'] && d['F']=="新增"){ + def udata = new BasicDBObject() + udata.create_time = new Date() + udata.province = d['G'] + udata.city = d['I'] + udata.district = d['J'] + udata.level = "省级" + udata.name = d['D'] + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + } + }) + } + def createDevMapping(){ def dbcursor = development_zone_CONN.find(new BasicDBObject()) for(def d : dbcursor){ @@ -548,6 +613,26 @@ class DevProcess { }); } + def deleteAddr2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d-> + if(d['Q'] && d['M']){ + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['Q'])) + def obj = development_zone_CONN.findOne(q) + if(obj){ + def q2 = new BasicDBObject() + q2.put("name", obj.name) + q2.put("address", d['I']+d['J']) + def sameObj = DEV_MAPPING_CONN.findOne(q2) + if(sameObj){ + DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id)) + } + } + } + + }); + } + def synchDevTo43(){ def dbcursor = development_zone_CONN.find() def rets = [] @@ -559,17 +644,19 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx",{d-> - if(d['P'] && d['P']=="路正常" && d['O']){ + ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d-> + if(d['R']){ def q = new BasicDBObject() - q.put("name", d['O']) + q.put("name", d['R']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['Q'] = obj._id.toString() - d['R'] = obj.name - def address = d['M']+d['N'] + d['P'] = obj._id.toString() + d['Q'] = obj.name + def address = d['L']+d['M'] q.put("address", address) +// def street = d['G'] +// q.put("township", street) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ def udata = new BasicDBObject() @@ -583,11 +670,110 @@ class DevProcess { udata.put("name",obj.name) udata.put("type", "地址") - udata.put("address", address) + udata.put("address",address) +// udata.put("type", "乡镇街道") +// udata.put("township", street) DEV_MAPPING_CONN.save(udata) + println obj.name +" "+address } } } },0); } + + + + def matchDevTownship(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> + if(d['C']){ + def q = new BasicDBObject() + q.put("name", d['C']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['E'] = obj._id.toString() + def street = d['D'] + q.put("township", street) + def sameObj = DEV_MAPPING_CONN.findOne(q) + if(!sameObj){ + def udata = new BasicDBObject() + udata.put("province", obj.province) + if(obj.city){ + udata.put("city", obj.city) + } + if(obj.district){ + udata.put("district", obj.district) + } + + udata.put("name",obj.name) +// udata.put("type", "乡镇街道") +// udata.put("township", street) + + udata.put("type", "区域") + udata.put("dev_district", street) + udata.put("c_date",new Date()) + DEV_MAPPING_CONN.save(udata) + println obj.name +" "+street + } + } + } + },1); + } + + def addAlias(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d-> + def q = new BasicDBObject() + q.put("name", d['A']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['D'] = obj._id.toString() + def arrs = d['B'].split("、") + def alias = obj.alias + if(!alias){ + alias = [] + } + for(def s : arrs){ + if(!alias.contains(s)){ + alias << s + } + } + + def udata = new BasicDBObject() + udata.put("alias", alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }) + } + + def addDevShape(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d-> + if(d['W']){ + def s = d['S'] + JSONObject jsonObject = JSONObject.fromObject(s); + Map map = DataHandleUtils.JsonToMap(jsonObject); + + def udata = new BasicDBObject() + udata.put("shape", map) + development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata)) + } + }) + } + + def addDevStreet(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> + if(d['J']=="是"){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道")) + for(def obj : dbcursor){ + def o = new BasicDBObject() + o.put("province" , obj.province) + o.put("city" , obj.city) + o.put("district" , obj.mzb_district_name) + o.put("name" , d['E']) + o.put("type" , "乡镇街道") + o.put("township" , obj.street) + DEV_MAPPING_CONN.save(o) + } + } + }) + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy new file mode 100644 index 000000000..8edf46edf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -0,0 +1,317 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PoiLocation { + + def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity() + def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity() + def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges() + def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity() + + static main(args) { + def task = new PoiLocation(); + task.run(); + } + def run(){ + def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"] + for(def city : citys){ + findPois(city) + } + } + + + def findPois(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("run_status", new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("_id", 1) + keys.put("province", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("township", 1) + keys.put("location", 1) + + def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def rets = [] + def index = 0 + for(def d : dbcursor){ + rets << d + if(rets.size()>=10000){ + index++ + def datas = checkData(index, rets) +// updateToEs(datas) + updateToEs2(datas) + updateToMG(datas) + rets = [] + print "." + } + } + return rets; + } + + def checkData(def index, def datas){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + + def updateToMG(rows){ + def rets = [] + for(def i=0;i + if(k!="id"){ + udata.put(k,v) + } + } + def q = new BasicDBObject() + q.put("_id", m.id) + if(udata){ + MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) + }else{ + println m.toString()+" 无法修改-无值" + } + + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy new file mode 100644 index 000000000..4d77b4cdf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels + +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class Test{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) +// 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + + static main(args) { + def task = new Test() + task.run() + } + + def run(){ + def i=1 + while(i<2){ + def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); + def rets = process(districts) + println i + " "+districts.size() +" "+rets.size() + i++ + } + + } + + def process(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+d.town+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + + + def updateQuery = new BasicDBObject("city", city).append("street",name) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + + def checkTown(){ + def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + for(def ad : dbcursor){ + def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street) + def count = TOWNPOLYLINE_COLL.count(polyline_q) + if(count==1){ + println ad.city+" "+ ad.street+" "+count + + def d = TOWNPOLYLINE_COLL.findOne(polyline_q) + if(d){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("polyline_source", "区域+街道匹配") + + def updateQuery = new BasicDBObject("_id", ad._id) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + } + } + + + def updateTownshipByRegion(){ + def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域") + def keys = new BasicDBObject() + keys.put("mzb_district_name",1) + keys.put("mzb_district_code",1) + + keys.put("city",1) + keys.put("district",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys) + for(def d : dbcursor){ + def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city) + def udata = new BasicDBObject() + udata.put("mzb_district_name",d.mzb_district_name) + udata.put("mzb_district_code",d.mzb_district_code) + ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true) + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy new file mode 100644 index 000000000..f6e830017 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -0,0 +1,324 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TownshipStrandUpdate { + + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + def fullCityMap = null + def regionMap = null + def commMap = null + def polylineMap = null + def newAdvMappingMap = null + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new TownshipStrandUpdate() + task.process() + + } + + + void process() { + fullCityMap = CommonDatautils.queryFullNameByCity() + regionMap = CommonDatautils.queryDistNameByCity() + commMap = CommonDatautils.queryCommunityByCity() + polylineMap = CommonDatautils.findAllTownRanges() + newAdvMappingMap = CommonDatautils.queryNewTownByCity() +// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + for(def city : citys){ + if(["中山"].contains(city)){ + continue + } + def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + if(!file.exists()){ + def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) + dists = checkData(dists) + def datas = createDatas(dists) + ExcelUtils.write(fileName, datas, "${city}") + } + } + } + + def checkData(def dists){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i maxScore){ + maxScore = score + town = townObj + } + } + if(town){ + town.score = maxScore + } + return town + } + + + def getSimilarTown(regions, fullCity, townstr){ + def town = null + for(def r : regions){ + def townList = commMap.get(fullCity.city+"#"+r) + + for(def townObj : townList){ + if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){ + town = townObj + }else if(townObj.alias && townObj.alias.contains(townstr)){ + town = townObj + }else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){ + def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","") + if(townstr.contains(newTown)){ + town = townObj + } + } + if(town){ + break + } + } + } + return town + } + + + def getFullRegion(fullCity, d){ + def regions = regionMap.get(fullCity.city) + def names = [] + if(regions){ + for(def r : regions){ + if(r.mzb_district_name.contains(d.region)){ + names << r.district + } + } + } + return names; + } + + + def createDatas(dists){ + def rets = [] + rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"] + for(def d : dists){ + rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score] + } + return rets; + } + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy index 7c9472e42..7d34463a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -1,13 +1,46 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection class TradeDatasDistBatch { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com") static main(args) { -// def + def task = new TradeDatasDistBatch() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d-> + + def udata = new BasicDBObject() + udata.put("city", d['A']) + udata.put("region", d['B']) + udata.put("name", d['C']) + udata.put("alias", d['D']) + alias_fang_COLL.save(udata) +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// def flag = null +// if(!dist){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district") +// if(rets && rets.size()==1){ +// flag = "别名匹配" +// dist = rets[0] +// } +// }else{ +// flag = "名称匹配" +// } +// if(flag){ +// d['F'] = dist.city +// d['G'] = dist.region +// d['H'] = dist.name +// } + }) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 71132614c..0bdd0555f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -354,7 +354,7 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" + def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 2b8cde742..50f070a3a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -72,7 +72,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); - def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"); + def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); + + // 最大重试次数 + private static final int MAX_TRY_COUNT = 5; static main(args) { runBatch(new FjwBigdataForsaleBatch(), args); @@ -86,12 +89,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def startDate = lastMonth+"18" - def endDate = curMonth+"19" + def endDate = curMonth+"19" //正式运行时curMonth要改 println "房源时间:"+startDate+" "+endDate - def houseIdsfilter = findAllHouseIdsByCity() + def houseIdsfilter = findAllHouseIdsByCity() for(def city : citys){ index++ @@ -133,7 +135,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ println city +" 小区量:"+districts.size() - + BloomFilter curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); def retMap = new ConcurrentHashMap() for(def i=0;i - if(!row['S'] && row['D']){ + if(!row['Y'] && row['G']){ def city = row['C'] - def region = row['D'] - def name = row['F'] - def address = row['H'] + def region = null //row['D'] + def name = row['E'] + def address = row['G'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name) + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) if(jsonObject != null && jsonObject.precise!=-1) { - row['S'] = jsonObject.get("id") - row['T'] = jsonObject.get("city") - row['U'] = jsonObject.get("district") - row['V'] = jsonObject.get("name") - row['W'] = jsonObject.get("propertyType") - row['X'] = jsonObject.get("credit") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") + row['AC'] = jsonObject.get("propertyType") +// row['X'] = jsonObject.get("credit") } } }) @@ -111,18 +111,18 @@ class MsbPocTest { def match(file){ ExcelUtils.multiThreadingModify(file, { row -> - if(!row['N']){ - def city = row['A'] - def region = row['B'] - def address = row['C'] //row['G']//row['F'] + if(!row['Y']){ + def city = row['C'] + def region = null //row['B'] + def address = row['G'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) + def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false) if(jsonObject != null) { - row['N'] = jsonObject.get("id") - row['O'] = jsonObject.get("city") - row['P'] = jsonObject.get("district") - row['Q'] = jsonObject.get("name") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") } } }) @@ -169,17 +169,17 @@ class MsbPocTest { def addCCBPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> - if(!row['H']){ + if(!row['V']){ // def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") // if(dist){ - if(row['D'] instanceof String){ - def d = CommonDatautils.findDistrictById(row['D']) + if(row['Y'] instanceof String){ + def d = CommonDatautils.findDistrictById(row['Y']) if(d){ def cityId = cityCodeDataMap.get(d.city) def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId) if(map){ - row['H'] = map.avg_price - row['I'] = map.calc_type + row['V'] = map.avg_price +// row['I'] = map.calc_type } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 29214f73f..f0c4282e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,17 +33,31 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\对标补充数据.xlsx" -// addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" + addGXDPrice(filePath) // addCcbPrice(filePath) - addThirdPrice(filePath) +// addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() +// addTradePrice(filePath) } + def addTradePrice(filePath){ + ExcelUtils.multiThreadingModify(filePath, { d -> + if(d['Y']){ + def dist = CommonDatautils.findDistrictById(d['Y']) + def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01') + if (tradePrice) { + d['W'] = tradePrice.toInteger() + } + } + + }); + } + def addRegion(){ ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d-> def dist = CommonDatautils.findDistrictById(d['A']) @@ -190,11 +204,11 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['G']) { - def dist = CommonDatautils.findDistrictById(d['G']) + if (!d['X']) { + def dist = CommonDatautils.findDistrictById(d['Y']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) - d['Z'] = thirdPrice + d['X'] = thirdPrice } } }) @@ -206,8 +220,8 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) - +// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) + q.put("s_date", 202404) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null for(def db : dbcursor){ @@ -227,12 +241,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['K']) { - def dist = CommonDatautils.findDistrictById(d['C']) + if (!d['M']) { + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['K'] = gxdPrice + d['M'] = gxdPrice } } }) @@ -256,16 +270,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['W']){ - if(d['G'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['G']) + if(!d['L']){ + if(d['B'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ - d['W'] = dbo.avg_price - d['X'] = dbo.cal_type -// d['J'] = dbo.avg_price_2b +// d['W'] = dbo.avg_price +// d['X'] = dbo.cal_type + d['L'] = dbo.avg_price_2b } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index 924db2def..35f821ce3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '上海市奉贤区星火开发区莲塘路251号8幢', + 'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index cfda2228d..63996e918 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) @@ -58,6 +60,9 @@ class GaoDePoiDataProcess extends BaseBatch{ def fullCityMap = null + DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL) static main(args) { runBatch(new GaoDePoiDataProcess(), args) @@ -69,12 +74,215 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchMapPoi(); - synchCompany() +// synchCompany() // poiCatProcess() // addressAndCatProcess() // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() +// sysnchVillage() + + addTownship() + } + + def addTownship(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d-> + if(d['E']){ +// def gaodeMap = getTownshipInfo(d['F']+","+d['E']) +// if(gaodeMap && gaodeMap.township){ +// if(gaodeMap.city && gaodeMap.city!="[]"){ +// d['H'] = gaodeMap.city +// } +// if( gaodeMap.district && gaodeMap.district!='[]'){ +// d['I'] = gaodeMap.district +// } +// if(gaodeMap.township && gaodeMap.township!='[]'){ +// d['J'] = gaodeMap.township +// } +// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){ +// d['K'] = gaodeMap.format_addr +// } +// +// } + + def q = new BasicDBObject() + q.put("city", d['H']) + q.put("district", d['I']) + q.put("street",d['J']) + def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q) + if(obj){ + d['L'] = "是" + } + } + },0) + + } + + def sysnchVillage(){ + def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"] + for(def city : buducitys){ + def pois = findAllVillagePoiMap(city) + def rows = createVillageData(city, pois) + + save(city, rows) + saveToMG(city, rows) + println city +" "+pois.size()+" add:"+rows.size() + } + + } + + def createVillageData(city, pois){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(pois.size()) + def townMap = findTownRange(city) + + for(def i=0;i + if(row['Q'] && row['Q']=='是'){ + def udata = new BasicDBObject() + udata.put("township", row['J']) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata)) + } + }) + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 19c0faebf..907e2a572 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -98,6 +98,23 @@ class InsidePolygon { } return list; } + + public static List> createPoint4(def pointList){ + List> alllist = new ArrayList<>(); + for(def pointArr : pointList){ + List list = new ArrayList<>(); + for(def arr : pointArr){ + Point point = new Point(); + double x = arr[0]; + double y = arr[1]; + point.setLon(x); + point.setLat(y); + list.add(point); + } + alllist << list + } + return alllist + } /** * 判定一个经纬度点与区域的关系 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index f14bedadd..6aba12207 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -76,7 +76,6 @@ class ElasticUtils { connection.close() } - static RestHighLevelClient getClient(String host) { RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { @@ -101,6 +100,30 @@ class ElasticUtils { } + static RestHighLevelClient getClient89(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + /** * 创建一个ES批处理器 * @param host diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 7e85c66ad..50882747b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 5 + def nThread = Runtime.getRuntime().availableProcessors() * 4 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index ab5c90233..b3b119384 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995 /** * 经纬度转地址
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java index 8cd9fc3f9..04e5d5de2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java @@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery; import com.fangjia.dal.SortParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank; +import com.fangjia.dal.mongo.model.AliasFang; import com.fangjia.dal.mongo.model.DistrictFacilitys; import com.fangjia.dal.seaweed.model.District; import com.fangjia.service.bo.DistrictInfo; @@ -540,10 +541,10 @@ public interface DistrictService extends ModeService { * * @param city * @param region - * @param masterDistrictNname + * @param masterDistrictName * @return */ - List findSlaves(String city, String region, String masterDistrictNname); + List findSlaves(String city, String region, String masterDistrictName); /** * 分期操作时设置此小区为人工操作标识 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index 31b588538..dd625b469 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService { } @Override - public List findSlaves(String city, String region, String masterDistrictNname) { + public List findSlaves(String city, String region, String masterDistrictName) { List params = new ArrayList(); params.add(new QueryParam("visible", 0)); params.add(new QueryParam("city", city)); params.add(new QueryParam("region", region)); - params.add(new QueryParam("masterDistrictNname", masterDistrictNname)); + params.add(new QueryParam("masterDistrictName", masterDistrictName)); return districtDao.findAllByParams(params); } @@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService { params.add(new QueryParam("$sr", new SrQuery(query))); return districtDao.findAllByParams(params); } - } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java new file mode 100644 index 000000000..c3136da96 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java @@ -0,0 +1,23 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public interface AliasFangService { + + /** + * 查询搜房别名与名称关系 + * + * @param city + * @param region + * @param names + * @return + */ + List findFangByNameAndAlias(String city, String region, Set names); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java new file mode 100644 index 000000000..2437795cd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java @@ -0,0 +1,27 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.dao.AliasFangDao; +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public class AliasFangServiceImpl implements AliasFangService{ + + private AliasFangDao aliasFangDao; + + @Override + public List findFangByNameAndAlias(String city, String region, Set names) { + return aliasFangDao.findFangByNameAndAlias(city, region, names); + } + + public void setAliasFangDao(AliasFangDao aliasFangDao) { + this.aliasFangDao = aliasFangDao; + } +}