Merge remote-tracking branch 'origin/master'

This commit is contained in:
mjt
2024-06-19 13:39:42 +08:00
39 changed files with 2403 additions and 302 deletions
@@ -0,0 +1,16 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.AliasFang;
import java.util.List;
import java.util.Set;
/**
* @author shenjian
* @since 2024/6/11
*/
public interface AliasFangDao extends MongoDao<AliasFang> {
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
}
@@ -0,0 +1,40 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.mongo.model.AliasFang;
import com.fangjia.dal.mongo.model.ConsumerAddress;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import org.apache.commons.lang.StringUtils;
import org.sothis.core.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
public class AliasFangDaoImpl extends MongoDaoImpl<AliasFang> implements AliasFangDao{
@Override
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
DBObject query = new BasicDBObject();
if (StringUtils.isNotBlank(city)) {
query.put("city", city);
}
if (StringUtils.isNotBlank(region)) {
query.put("region", region);
}
// db.alias_fang.find({'city': 'ÖØÇì', 'region': 'Óå±±', $or: [{'name': 'ÖйúĦ'}, {'alias': 'ÖйúĦ'}]})
if(CollectionUtils.isNotEmpty(names)) {
BasicDBList dbList = new BasicDBList();
dbList.add(new BasicDBObject("name", new BasicDBObject("$in", names)));
dbList.add(new BasicDBObject("alias", new BasicDBObject("$in", names)));
query.put("$or", dbList);
}
List<AliasFang> objs = find(query, null, null, 0, Integer.MAX_VALUE);
return objs;
}
}
@@ -0,0 +1,68 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
/**
* ËÑ·¿Ãû³ÆÓë±ðÃû¹ØÏµ
*/
@Entity
@Table(name = "business.alias_fang")
public class AliasFang implements MongoEntity {
private String id;
private String city;
private String region;
private String name;
private String alias;
@Override
@Id
@GeneratedValue
@Column(name = "_id", unique = true, nullable = false)
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "alias")
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
}
@@ -187,7 +187,7 @@ public class District implements SeaweedEntity {
private double score;
// 小区性质
private String landProperty;
private String masterDistrictNname;
private String masterDistrictName;
private String ip;
private String addrType;
private boolean selected;
@@ -309,12 +309,12 @@ public class District implements SeaweedEntity {
}
@Column(name = "master_district_name")
public String getMasterDistrictNname() {
return masterDistrictNname;
public String getMasterDistrictName() {
return masterDistrictName;
}
public void setMasterDistrictNname(String masterDistrictNname) {
this.masterDistrictNname = masterDistrictNname;
public void setMasterDistrictName(String masterDistrictName) {
this.masterDistrictName = masterDistrictName;
}
public void setCategory(String category) {
@@ -391,13 +391,106 @@ public class AddrSvcController {
}
};
/**
* 涉农识别-新规则-南京银行
*
* @param model
* @return
*/
public Object njcbVillageIdentifyAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
String address = model.getAddress();
params.put("address", address);
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
params.put("type", type);
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
//我们的结果
JSONObject result = json.getJSONObject("result");
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
//结果比较,如果涉农结果不一致
//调用齐梁 type :1 对公 0 对私
//我们 type :1 对私 2 对共
String qltype = "2".equals(type) ? "1" : "0";
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
//城市不一样 用我们
if (result.containsKey("city")) {
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
basicModel.setResult(result);
}
} else {
basicModel.setResult(result);
}
} catch (Exception e) {
e.printStackTrace();
basicModel.setResult(result);
}
}
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 涉农识别-新规则-基础版本
*
* @param model
* @return
*/
public Object villageIdentifyAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
@@ -435,11 +528,8 @@ public class AddrSvcController {
if (json.containsKey("result")) {
//我们的结果
JSONObject result = json.getJSONObject("result");
// 如果是无效信息,则用我们的
if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) {
basicModel.setResult(result);
}//我们省级及以上有开发区,直接用我们的结果
else if (result.containsKey("devDistrictName")) {
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else {
@@ -233,10 +233,13 @@ public class ComprehensiveQueryController {
communityAvgPriceParams.put("city", model.getCity());
communityAvgPriceParams.put("address", model.getAddress());
communityAvgPriceParams.put("category", model.getCategory());
communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
communityAvgPriceParams.put("suggestName", String.valueOf(model.getName()));
communityAvgPriceParams.put("preciseLevel", "strict");
communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB");
JSONObject matchResponse = null;
try {
matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8");
// matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
} catch (HttpRequestException e) {
@@ -256,9 +259,21 @@ public class ComprehensiveQueryController {
DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district));
if (null != statBo) {
if (statBo.getDistrictStat() != null) {
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
// 有2B价格,使用2B价格
Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b();
if (avgPrice2b != null && avgPrice2b > 0) {
dto.setAvgPrice(avgPrice2b);
} else {
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
}
} else if (statBo.getVillaDistrictStat() != null) {
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
// 有2B价格,使用2B价格
Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b();
if (avgPrice2b != null && avgPrice2b > 0) {
dto.setAvgPrice(avgPrice2b);
} else {
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
}
}
}
@@ -638,10 +653,18 @@ public class ComprehensiveQueryController {
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
// 有2B价格用2B价格,没有用2C
Integer minPrice = 0;
Integer maxPrice = 0;
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) {
minPrice = (int) (evaluation.getAvgPrice2b() * 0.95);
maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05);
} else {
minPrice = (int) (evaluation.getAvgPrice() * 0.95);
maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
}
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
@@ -657,7 +680,11 @@ public class ComprehensiveQueryController {
districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName());
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) {
districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b());
} else {
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
}
// 根据城市等 级设置D级可信度
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
@@ -882,7 +909,11 @@ public class ComprehensiveQueryController {
otherPriceRets = new HashMap();
}
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){
otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b());
}else {
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
}
Map<String, Object> levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit());
Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit);
@@ -51,7 +51,7 @@ public class GdBankController {
};
/**
* 涉农识别-新规
* 涉农识别-新规则-光大银行
*
* @param model
* @return
@@ -95,8 +95,8 @@ public class GdBankController {
} else if (("否").equals(result.get("countyLevelCity"))) {
result.put("prefectureLevelCity", "是");
}
// 如果是无效信息,则用我们的
if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) {
// 如果是无效信息或者没有行政区划,则用我们的
if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) {
basicModel.setResult(result);
//我们省级及以上有开发区,直接用我们的结果
} else if (result.containsKey("devDistrictName")) {
@@ -10,17 +10,7 @@ import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -30,6 +20,7 @@ import javax.validation.Valid;
import com.fangjia.core.util.*;
import com.fangjia.dal.*;
import com.fangjia.dal.mongo.dao.AliasFangDao;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
@@ -39,6 +30,7 @@ import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.open.dto.addressGovernance.StructureDto;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.address.AddressParserUtil;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
@@ -138,7 +130,6 @@ import com.mongodb.DBObject;
@Authority(role = Role.USER)
public class PropertyController {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
private DistrictService districtService;
private AddressParser addressParser;
@@ -186,7 +177,7 @@ public class PropertyController {
private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20;
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" };
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" };
static final int BAI_DU_GEO = 1;// 百度系坐标
static final int GAO_DE_GEO = 2;// 高德系坐标
@@ -196,6 +187,7 @@ public class PropertyController {
private DistrictRoomSqmetreRentService districtRoomSqmetreRentService;
private RentDealPriceService rentDealPriceService;
private EvaluateRatioService evaluateRatioService;
private AliasFangService aliasFangService;
private static final int EVALUATE_SHOW_DETAIL = 1;
private static final Map<String, String> PROPERTY_TYPES = new HashMap<String, String>() {
@@ -609,6 +601,25 @@ public class PropertyController {
dto.setBlock(district.getBlock());
}
dto.setPropertyType(district.getEstateType2());
if (CollectionUtils.isNotEmpty(district.getAlias())) {
// 将最相近的别名输出给用户
String alias = null;
double similarity = 0;
if (StringUtils.isNotBlank(model.getKeyword())) {
for (String mappingAlias : district.getAlias()) {
double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword());
if (curr > similarity ) {
similarity = curr;
alias = mappingAlias;
}
}
}
if (StringUtils.isNotBlank(alias)) {
dto.setAlias(alias);
} else {
dto.setAlias(district.getAlias().get(0));
}
}
list.add(dto);
}
@@ -2516,7 +2527,7 @@ public class PropertyController {
model.setMax(100);
}
List<String> districtNames = new ArrayList<String>();
Set<String> districtNames = new HashSet<String>();
if (null == district) {
districtNames.add(model.getName());
} else {
@@ -2524,14 +2535,56 @@ public class PropertyController {
if (district.getAlias() != null) {
districtNames.addAll(district.getAlias());
}
}
List<DealPrice> dealPrices = null;
for (String name : districtNames) {
dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
if (!dealPrices.isEmpty()) {
break;
final String masterDistrictName = district.getMasterDistrictName();
if (StringUtils.isNotBlank(masterDistrictName)) {
List<District> districts;
if ("master".equals(masterDistrictName)) {
// 所有的分期数据
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName());
} else {
// 根据主小区名查询所有名称
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName);
}
if (CollectionUtils.isNotEmpty(districts)) {
for (District districtTemp : districts) {
districtNames.add(districtTemp.getName());
}
}
// districtNames 不包含 主小区名,加进去
if (!districtNames.contains(masterDistrictName)){
districtNames.add(masterDistrictName);
}
}
}
// 将搜房中的别名与名称加入查询集合
try {
List<AliasFang> aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames);
Set<String> aliasSet = new HashSet<String>();
for (AliasFang aliasFang : aliasFangs) {
if (StringUtils.isNotBlank(aliasFang.getName())) {
aliasSet.add(aliasFang.getName());
}
if (StringUtils.isNotBlank(aliasFang.getAlias())) {
aliasSet.add(aliasFang.getAlias());
}
}
districtNames.addAll(aliasSet);
} catch (Exception ex) {
ex.printStackTrace();
}
// List<DealPrice> dealPrices = new ArrayList<DealPrice>();
// 将主名与映射别名查得成交加入 存在不能查出最新成交问题
// for (String name : districtNames) {
// List<DealPrice> dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
// if (CollectionUtils.isNotEmpty(dealPricesTempList)) {
// dealPrices.addAll(dealPricesTempList);
// }
// }
List<DealPrice> dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList<String>(districtNames), model.getMax() * 5, model);
if (model.getSize() != null) {
Collections.sort(dealPrices, new Comparator<DealPrice>() {
@Override
@@ -2580,15 +2633,9 @@ public class PropertyController {
}
deal = new LinkedHashMap<String, Object>();
dealPrice.setId(null);
if (district != null) {
deal.put("city", district.getCity());
deal.put("district", district.getRegion());
deal.put("name", district.getName());
} else {
deal.put("city", dealPrice.getCity());
deal.put("district", dealPrice.getRegion());
deal.put("name", dealPrice.getDistrictName());
}
deal.put("city", dealPrice.getCity());
deal.put("district", dealPrice.getRegion());
deal.put("name", dealPrice.getDistrictName());
if (dealPrice.getArea() == null) {
deal.put("size", "");
} else {
@@ -3501,6 +3548,33 @@ public class PropertyController {
return null;
}
private List<DealPrice> findDealPricesByNames(String city, String region, List<String> names, int max,
DealPriceModel model) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("city", city));
params.add(new QueryParam("region", region));
params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names));
if (model.getRoom() != null) {
params.add(new QueryParam("room", model.getRoom()));
}
if (model.getHall() != null) {
params.add(new QueryParam("hall", model.getHall()));
}
// 18年之后抓住的数据没有score字段
// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l));
List<DealPrice> dealPrices = dealPriceService.findDealPrices(params, max);
List<DealPrice> distinctDealPrices = new ArrayList<DealPrice>();
for (DealPrice dealPrice : dealPrices) {
if (dealPrice.getAvgPrice() == null) {
dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea()));
}
if (!checkExists(distinctDealPrices, dealPrice)) {
distinctDealPrices.add(dealPrice);
}
}
return distinctDealPrices;
}
private List<DealPrice> findDealPricesByName(String city, String region, String name, int max,
DealPriceModel model) {
List<QueryParam> params = new ArrayList<QueryParam>();
@@ -5959,4 +6033,8 @@ public class PropertyController {
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
this.communityBasePriceService = communityBasePriceService;
}
public void setAliasFangService(AliasFangService aliasFangService) {
this.aliasFangService = aliasFangService;
}
}
@@ -15,6 +15,10 @@ public class PropertyDto {
private String propertyType;
private String block;
private String distance;
/**
* 新增小区别名
*/
private String alias;
public String getId() {
return id;
}
@@ -88,4 +92,12 @@ public class PropertyDto {
public void setDistance(String distance) {
this.distance = distance;
}
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
}
@@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -61,61 +62,67 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
def objectId = dateToObjectId(date)
println objectId
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
//
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
// println "历史id加载量:"+dealIds.size()
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
//// delPgData()
// saveToPgSql(rows)
//
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
//// saveToPgSql2(datas)
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
// delPgData()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
}
def readTxt(def fileName){
@@ -682,7 +689,13 @@ class BigdataDealBatch extends BaseBatch{
}
def updateCrawlTradeDate(){
def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20"
def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd")
def objectId = dateToObjectId(date)
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2)
def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1)
println(objectId)
CcbCommonUtils.updateTrades(startDate, endDate, objectId)
}
@@ -822,4 +835,15 @@ class BigdataDealBatch extends BaseBatch{
println "end: "+new Date()
}
def dateToObjectId(Date date) {
// 获取时间戳(单位:秒)
def timestamp = (date.time / 1000) as int
def hexTimestamp = Integer.toHexString(timestamp)
// 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值
def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000'
return new ObjectId(objectIdStr)
}
}
@@ -15,9 +15,10 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys_fjw_all = [ "乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
@@ -439,7 +440,7 @@ class CcbCommonUtils {
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","新泾三村")
// q.put("name","鸿运大院")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -711,4 +712,16 @@ class CcbCommonUtils {
return map
}
public static def updateTrades(startDate, endDate, cDate){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("_id", new BasicDBObject('$gte', cDate))
def count = DEAL_PRICE_COLL.count(q)
if(count>0){
def udata = new BasicDBObject()
udata.put("trade_date", endDate)
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
}
}
}
@@ -389,7 +389,7 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = CcbCommonUtils.citys_all
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){
@@ -1301,20 +1301,44 @@ ExcelUtils.multiThreadingModify(fileName, {d->
// d['H'] = dist._id.toString()
// }
// }
if(!d['A']){
if(!d['A'] && d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['K'] = dist.estate_type2
d['L'] = dist.status
d['M'] = dist.name_cat
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
// d['H'] = dist.name
}
}
}
},0)
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
//ExcelUtils.multiThreadingModify(filePath, { d ->
// if(d['B']){
//
// def obj = CommonDatautils.findDistrictById(d['B'])
// if(obj){
// def markMap = obj.mark_map
// if(!obj.mark_map){
// markMap = [:]
// }
// def data = new BasicDBObject()
// markMap.put("exclusive", "是")
// data.put("mark_map", markMap)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
@@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
// }
// }
//
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
def size = regionDatas.size()
final CountDownLatch latch = new CountDownLatch(size)
def totalCount = new AtomicLong()
for(def i = 0;i<regionDatas.size();i++){
final def arr = regionDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
totalCount.addAndGet(1)
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
if(row){
rets << row
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// };
// })
}
latch.await()
// latch.await()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
def allDistRegionPreMap = [:]
def allLastMonthMap = [:]
def allLastYearMap = [:]
def allCcbPriceMap = [:]
def allCcbPriceCaseCountMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
allDistricts.addAll(districts)
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
allPriceMap = allPriceMap + priceMap
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
// allPriceMap = allPriceMap + priceMap
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
allRentMap = allRentMap + rentMap
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceMap = allCcbPriceMap + datasMap
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
def cityRankMap = findCommunityCityRank(datas)
allCityRankMap = allCityRankMap + cityRankMap
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
allLastYearMap = allLastYearMap + lastYearMonthMap
}
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(cityLists){
rets.addAll(cityLists)
}
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
final CountDownLatch latch = new CountDownLatch(distCount)
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(row){
rets << row
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def sellDataType = 1
def rentDataType = 1
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
//房源查询是否按建行均价筛选房源
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
def sell_avg_price = null
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def sell_count = 0
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
if(allCcbPriceMap.containsKey(dist._id.toString())){
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
}
}
if(sell_count){
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
}
}
}
if(sellMap){
sell_count = sellMap.avg_price.size()
if(sell_count>3){
sell_avg_price = avgValue(sellMap.avg_price)
if(sellMap){
sell_avg_area = avgValue(sellMap.area)
sell_avg_total = avgValue(sellMap.total_price)
}else{
sell_count = 0
sellDataType = null
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// if(sell_count>3){
// if(allCcbPriceMap.containsKey(dist._id.toString())){
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
// }else{
// sell_count = 0
// sellDataType = null
// }
// }else{
// sell_count = 0
// sellDataType = null
// }
}
}
def rent_avg_price = null
def rent_avg_area = null
def rent_avg_total = null
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
rentDataType = null
}
}
if(!sell_avg_price){
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
// if(!sell_avg_price){
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
// }
if(!rent_avg_price){
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def avgPrice = null
if(region){
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
}else{
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
}
if (sell_count < 10) {
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
sellStartDate = startDate
}
if (sell_count) {
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
}
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
return datas;
}
def findCommunityPriceByCityMap(cityCode, month){
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
})
return dataMap;
}
def findCommunityPriceCountByCityMap(cityCode, month){
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
if(r.gujiabdsrt3mlgcasenum){
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
}
})
return dataMap;
}
def findCcbCityPrice(cityCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def avgPrice = null
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
return regionPriceMap;
}
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
def price = null
pgsqlInstance.eachRow(sql, { r ->
price = r.gujia_bds_clc_avprc
})
return price;
}
def districtCityPrictRate(cityPrice, datas){
def distCityRateMap = [:]
@@ -13,11 +13,13 @@ import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.poi.ss.util.RegionUtil
import org.bson.Document
import org.bson.types.ObjectId
@@ -117,6 +119,7 @@ class CommonDatautils {
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -274,7 +277,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","揭흩소蹈")
// q.put("name","托斯卡纳庄园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -283,7 +286,7 @@ class CommonDatautils {
}
}
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -4571,6 +4574,28 @@ class CommonDatautils {
return townMap
}
static def findAllTownRanges(){
def q = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("town", 1)
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
def townMap = [:]
for(def d : dbcursor){
def lists = townMap.get(d.city)
if(!lists){
lists = []
}
def t = [:]
t.region = d.region
t.town = d.town
lists << t
townMap.put(d.city, lists)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
@@ -4605,6 +4630,108 @@ class CommonDatautils {
}
static def queryFullNameByCity(){
def q = new BasicDBObject()
q.put("type","城市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def cityMap = [:]
for(def d : dbcursor){
def city = RegionData.formatCity(d.city)
if(city){
cityMap.put(city, d)
}else{
//println d.city +" 格式化不出"
}
}
return cityMap
}
static def queryDistNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
def regions = regionMap.get(d.city)
if(!regions){
regions = []
}
regions << d
regionMap.put(d.city, regions)
}else{
println d.province+" "+d.district +" 格式化不出民政部区域"
}
}
return regionMap
}
static def queryCommunityByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city
def list = townMap.get(key)
if(!list){
list = []
}
def t = [:]
t.region = d.district
t.town = d.street
list << t
townMap.put(key, list)
}
return townMap
}
static def queryDistNameCity(){
def q = new BasicDBObject()
@@ -4675,6 +4802,54 @@ class CommonDatautils {
}
static def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
@@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -24,13 +26,14 @@ class DevProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
// task.devSplitRoad()
task.devNameSplitRoad()
task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -44,7 +47,15 @@ class DevProcess {
// task.addNewDev()
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo43()
// task.addNewDev2()
// task.addAlias()
// task.addDevShape()//添加开发区围栏
// task.matchDevTownship()//添加乡镇街道
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
}
@@ -162,69 +173,101 @@ class DevProcess {
}
}
def findDev(city, name){
def q = new BasicDBObject()
q.put("name", name)
q.put("city", city)
def obj = development_zone_CONN.findOne(q)
return obj
}
def devSplitRoad(){
def i = 10;
while(i<11){
def fileName = "/home/goojia/gj/company-${i}.xlsx"
def i = 0;
while(i<20){
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\company-${i}.xlsx"
fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx"
}
ExcelUtils.multiThreadingModify(fileName,{d->
/*if(d['C'] && !d['V']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['V'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['W'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['X'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['Y'] = jsonObject.get("street")
if(streetNumber!='null'){
d['Z'] = jsonObject.get("streetNumber")
}
}
}*/
if(d['C'] && d['B']!='河北省'){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C'])
if(jsonObject){
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['J'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['K'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['L'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['M'] = jsonObject.get("street")
if(streetNumber!='null'){
d['N'] = jsonObject.get("streetNumber")
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['E'] && !d['G']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
if(jsonObject1) {
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
def devName = jsonObject1.get("dev")
def obj = findDev(d['B'], devName)
d['F'] = devName
if (obj) {
d['Q'] = obj._id.toString()
d['R'] = obj.name
}
}
}
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
d['O'] = jsonObject.get("dev")
}
}
}
},0)
i++
if(d['C'] && !d['S']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
d['G'] = jsonObject.get("province")
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
d['H'] = jsonObject.get("city")
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
d['I'] = jsonObject.get("district")
}
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
d['J'] = jsonObject.get("township")
}
if (jsonObject && street && streetNumber && street != "null") {
d['K'] = jsonObject.get("street")
def road = jsonObject.get("street")
if (streetNumber != 'null') {
d['L'] = jsonObject.get("streetNumber")
road = road + jsonObject.get("streetNumber")
} else {
d['N'] = "信息不完整"
}
if (!d['N'] && !d['C'].contains(road)) {
d['N'] = "原地址不含路"
}
if (!d['N']) {
def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$')
if (!flag) {
d['N'] = "路非法"
}
def roadsize = jsonObject.get("street").size()
if (!d['N'] && (roadsize < 3 || roadsize > 7)) {
d['N'] = "路太长|路太短"
}
}
}else{
d['N'] = "无地址"
}
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
def devName = jsonObject.get("dev")
def obj = findDev(d['B'], devName)
d['M'] = devName
if (obj) {
d['S'] = obj._id.toString()
d['T'] = obj.name
}
}
}
},0)
i++
}
}
}
@@ -234,7 +277,7 @@ class DevProcess {
def devNameSplitRoad(){
def fileName = "/home/goojia/gj/company.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx"
fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx"
}
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['P'] && d['P']=="·Õý³£" && !d['O']){
@@ -291,17 +334,18 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
if(d['H']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
if(!d['Q'] && d['L']){
def q = new BasicDBObject()
q.put("name", d['H'])
q.put("name", d['L'])
q.put("city",d['G'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['S'] = obj._id.toString()
d['T'] = obj.name
d['Q'] = obj._id.toString()
d['R'] = obj.name
}
}
},1);
},0);
}
def calAddrDistince(){
@@ -506,6 +550,27 @@ class DevProcess {
})
}
def addNewDev2(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
if(d['F'] && d['F']=="新增"){
def udata = new BasicDBObject()
udata.create_time = new Date()
udata.province = d['G']
udata.city = d['I']
udata.district = d['J']
udata.level = "省级"
udata.name = d['D']
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
if(!obj){
development_zone_CONN.save(udata)
}else{
println udata.name +" 已存在"
}
}
})
}
def createDevMapping(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
for(def d : dbcursor){
@@ -548,6 +613,26 @@ class DevProcess {
});
}
def deleteAddr2(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d->
if(d['Q'] && d['M']){
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['Q']))
def obj = development_zone_CONN.findOne(q)
if(obj){
def q2 = new BasicDBObject()
q2.put("name", obj.name)
q2.put("address", d['I']+d['J'])
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(sameObj){
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id))
}
}
}
});
}
def synchDevTo43(){
def dbcursor = development_zone_CONN.find()
def rets = []
@@ -559,17 +644,19 @@ class DevProcess {
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx",{d->
if(d['P'] && d['P']=="路正常" && d['O']){
ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
if(d['R']){
def q = new BasicDBObject()
q.put("name", d['O'])
q.put("name", d['R'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['Q'] = obj._id.toString()
d['R'] = obj.name
def address = d['M']+d['N']
d['P'] = obj._id.toString()
d['Q'] = obj.name
def address = d['L']+d['M']
q.put("address", address)
// def street = d['G']
// q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){
def udata = new BasicDBObject()
@@ -583,11 +670,110 @@ class DevProcess {
udata.put("name",obj.name)
udata.put("type", "µØÖ·")
udata.put("address", address)
udata.put("address",address)
// udata.put("type", "乡镇街道")
// udata.put("township", street)
DEV_MAPPING_CONN.save(udata)
println obj.name +" "+address
}
}
}
},0);
}
def matchDevTownship(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
if(d['C']){
def q = new BasicDBObject()
q.put("name", d['C'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['E'] = obj._id.toString()
def street = d['D']
q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){
def udata = new BasicDBObject()
udata.put("province", obj.province)
if(obj.city){
udata.put("city", obj.city)
}
if(obj.district){
udata.put("district", obj.district)
}
udata.put("name",obj.name)
// udata.put("type", "乡镇街道")
// udata.put("township", street)
udata.put("type", "区域")
udata.put("dev_district", street)
udata.put("c_date",new Date())
DEV_MAPPING_CONN.save(udata)
println obj.name +" "+street
}
}
}
},1);
}
def addAlias(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
def q = new BasicDBObject()
q.put("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['D'] = obj._id.toString()
def arrs = d['B'].split("、")
def alias = obj.alias
if(!alias){
alias = []
}
for(def s : arrs){
if(!alias.contains(s)){
alias << s
}
}
def udata = new BasicDBObject()
udata.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
})
}
def addDevShape(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d->
if(d['W']){
def s = d['S']
JSONObject jsonObject = JSONObject.fromObject(s);
Map<String, Object> map = DataHandleUtils.JsonToMap(jsonObject);
def udata = new BasicDBObject()
udata.put("shape", map)
development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata))
}
})
}
def addDevStreet(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
if(d['J']=="是"){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道"))
for(def obj : dbcursor){
def o = new BasicDBObject()
o.put("province" , obj.province)
o.put("city" , obj.city)
o.put("district" , obj.mzb_district_name)
o.put("name" , d['E'])
o.put("type" , "乡镇街道")
o.put("township" , obj.street)
DEV_MAPPING_CONN.save(o)
}
}
})
}
}
@@ -0,0 +1,317 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
static main(args) {
def task = new PoiLocation();
task.run();
}
def run(){
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
for(def city : citys){
findPois(city)
}
}
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("run_status", new BasicDBObject('$exists', false))
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
def index = 0
for(def d : dbcursor){
rets << d
if(rets.size()>=10000){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
updateToEs2(datas)
updateToMG(datas)
rets = []
print "."
}
}
return rets;
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
rets << map
if(map.update_status && map.update_status!=100){
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
}
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def running(d){
def regions = getFullRegion(d)
def town = null
def map =[:]
if(d.location){
//判断区域否在围栏里
def regionFlag = null
if(d.district){
for(def rs : regions){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
if(regionFlag){
//判断township是否在统计局中
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
if(town){
if(town.street == d.township){
//打标记
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = town.street
}
}else{
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
if (mapTown) {
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = mapTown.street
}else{
if(d.district!=gaodeMap.district){
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
}
}
}else{
//打标记“经纬度有误,不在区域围栏中”
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
return map;
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.district)){
names << r
}
}
}
return names;
}
def getSimilarTown(regions, city, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.update_status && map.update_status==2){
map.remove("run_status")
map.remove("update_status")
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def updateToMG(rows){
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def udata = new BasicDBObject()
m.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
def q = new BasicDBObject()
q.put("_id", m.id)
if(udata){
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
}else{
println m.toString()+" 无法修改-无值"
}
}
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
}
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
}
@@ -50,7 +50,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
synchTown()
// synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -61,6 +61,13 @@ class TownMiddleProcess {
// addDistrictDiffFlag()
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
}
@@ -800,4 +807,126 @@ class TownMiddleProcess {
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
});
}
def updateCommunityDiffStreet(){
def q = new BasicDBObject("type", "乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
def query = new BasicDBObject()
query.put("city_name", d.city)
query.put("area_name", d.district)
query.put("street_name", d.street)
def obj = COMMUNITY_CONN.findOne(query)
if(!obj){
query.put("street_name", d.district+d.street)
obj = COMMUNITY_CONN.findOne(query)
if(obj){
def newStreet = obj.street_name.replace(d.district, "")
def udata = new BasicDBObject()
udata.put("street_name", newStreet)
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
}else{
println d.city +" "+d.district+" "+d.street
}
}
}
}
def updateTownPoline(){
def city = "唐山市"
def name = "高新技术开发区街道"
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("city", city).append("street",name)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
def checkTown(){
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def ad : dbcursor){
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
def count = TOWNPOLYLINE_COLL.count(polyline_q)
if(count==1){
println ad.city+" "+ ad.street+" "+count
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("polyline_source", "区域+街道匹配")
def updateQuery = new BasicDBObject("_id", ad._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
def updateTownshipByRegion(){
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
def keys = new BasicDBObject()
keys.put("mzb_district_name",1)
keys.put("mzb_district_code",1)
keys.put("city",1)
keys.put("district",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
for(def d : dbcursor){
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
def udata = new BasicDBObject()
udata.put("mzb_district_name",d.mzb_district_name)
udata.put("mzb_district_code",d.mzb_district_code)
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
}
@@ -0,0 +1,324 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class TownshipStrandUpdate {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def fullCityMap = null
def regionMap = null
def commMap = null
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new TownshipStrandUpdate()
task.process()
}
void process() {
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
for(def city : citys){
if(["中山"].contains(city)){
continue
}
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
}
}
}
def checkData(def dists){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets << running(d)
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return dists
}
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def gaodeMap = null
def town = null
def mapTown = null
def oldSimlarTown = null
def newSimlarTown = null
if(d.lng){
d.lng = d.lng/10000000000.0
d.lat = d.lat/10000000000.0
if(d.township){
town = getSimilarTown(regions, fullCity, d.township)
}
if(town && town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
d.polygon_town = town.street
}
}
d.strand_street = ""
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
if(mapTown){
d.strand_street = mapTown.street
if(mapTown.shape && mapTown.shape.coordinates){
d.map_town_shape = "有"
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
if(flag){
d.map_polygon_town = mapTown.street
}
}
}else{
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
if(oldSimlarTown){
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
d.old_similar_town_score = oldSimlarTown.score
}
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
if(newSimlarTown){
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
d.new_similar_town_score = newSimlarTown.score
}
}
}
}
def mapCity = null
def mapRegion = null
def mapTownship = null
def mapAddr = null
if(gaodeMap){
if(gaodeMap.city && gaodeMap.city!="[]"){
mapCity = gaodeMap.city
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
}
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
mapAddr = gaodeMap.format_addr
}
}
d.mapCity = mapCity
d.mapRegion = mapRegion
d.mapTownship = mapTownship
d.mapAddr = mapAddr
if(mapCity && mapRegion){
if(mapCity == fullCity.city){
if(regions.contains(mapRegion)){
if(mapTownship){
if(town){
d.old_tjj_township = "是"
}else{
d.old_tjj_township = "否"
}
if(mapTown){
d.new_tjj_township = "是"
}else{
d.new_tjj_township = "否"
}
if(mapTownship == d.township){
if(town){
d.remark = "相同且为统计局乡镇"
}else{
d.remark = "非统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}else{
if(mapTown){
d.remark = "更正"
}else{
d.remark = "不同且不为统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}
}else{
d.remark = "地图无乡镇街道"
d.opt = "地图无街道信息"
}
}else{
d.remark = "区域不同"
d.opt = "经纬度可能跨区域"
}
}else{
d.remark = "城市不同"
d.opt = "经纬度可能跨城市"
}
}
if(d.map_town_shape && d.map_town_shape=="有"){
if(!d.map_polygon_town){
d.opt = "经纬度可疑"
}
}
return d;
}
def getSimilarTownByStr(fullCity, townstr, townList){
def maxScore = 0
def town = null
for(def townObj : townList){
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
if(score > maxScore){
maxScore = score
town = townObj
}
}
if(town){
town.score = maxScore
}
return town
}
def getSimilarTown(regions, fullCity, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(fullCity.city+"#"+r)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def getFullRegion(fullCity, d){
def regions = regionMap.get(fullCity.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region)){
names << r.district
}
}
}
return names;
}
def createDatas(dists){
def rets = []
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
for(def d : dists){
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
}
return rets;
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
}
@@ -1,13 +1,46 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class TradeDatasDistBatch {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
static main(args) {
// def
def task = new TradeDatasDistBatch()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("alias", d['D'])
alias_fang_COLL.save(udata)
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def flag = null
// if(!dist){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }else{
// flag = "名称匹配"
// }
// if(flag){
// d['F'] = dist.city
// d['G'] = dist.region
// d['H'] = dist.name
// }
})
}
}
@@ -354,7 +354,7 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -72,7 +72,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
runBatch(new FjwBigdataForsaleBatch(), args);
@@ -86,12 +89,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19"
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
def houseIdsfilter = findAllHouseIdsByCity()
for(def city : citys){
index++
@@ -133,7 +135,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
println city +" 小区量:"+districts.size()
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
@@ -161,7 +163,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
if(!houseIdsfilter.mightContain(m.houseId)){
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
@@ -180,6 +182,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
curhousefilter.put(m.houseId)
houseIdsfilter.put(m.houseId)
}else{
println city +" "+m.houseId+" 已存在"
}
@@ -192,7 +196,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
} catch (Exception e) {
e.printStackTrace()
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
} finally {
latch.countDown()
}
@@ -268,7 +272,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
// def newRets = []
@@ -707,6 +711,33 @@ class FjwBigdataForsaleBatch extends BaseBatch{
return rets;
}
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
int retryCount = 0;
boolean success = false;
def rets = []
while (retryCount < MAX_TRY_COUNT && !success) {
try {
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
// 这里可以选择在重试之前等待一段时间
Thread.sleep(3000); // 例如等待1秒
}
}
if (!success) {
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
}
return rets;
}
/**
* 是否是别墅
* @return
@@ -49,14 +49,15 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = ["ÎÚ³ľÆë"]//CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def stop = false
def month = -1
while(!stop){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
if(sDate<202004){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
}
month = month -1
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
if(sDate == 201912){
@@ -13,7 +13,7 @@ class MsbPocTest {
task.run();
}
def run(){
def inputfile = "D:\\ĘýžÝ\\˝¨ĐĐ\\ˇż˛úšŔÖľ\\202311šăÖÝ.xlsx"
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
@@ -69,21 +69,21 @@ class MsbPocTest {
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['S'] && row['D']){
if(!row['Y'] && row['G']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['H']
def region = null //row['D']
def name = row['E']
def address = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['S'] = jsonObject.get("id")
row['T'] = jsonObject.get("city")
row['U'] = jsonObject.get("district")
row['V'] = jsonObject.get("name")
row['W'] = jsonObject.get("propertyType")
row['X'] = jsonObject.get("credit")
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("propertyType")
// row['X'] = jsonObject.get("credit")
}
}
})
@@ -111,18 +111,18 @@ class MsbPocTest {
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['N']){
def city = row['A']
def region = row['B']
def address = row['C'] //row['G']//row['F']
if(!row['Y']){
def city = row['C']
def region = null //row['B']
def address = row['G'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null) {
row['N'] = jsonObject.get("id")
row['O'] = jsonObject.get("city")
row['P'] = jsonObject.get("district")
row['Q'] = jsonObject.get("name")
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
}
}
})
@@ -169,17 +169,17 @@ class MsbPocTest {
def addCCBPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['H']){
if(!row['V']){
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
// if(dist){
if(row['D'] instanceof String){
def d = CommonDatautils.findDistrictById(row['D'])
if(row['Y'] instanceof String){
def d = CommonDatautils.findDistrictById(row['Y'])
if(d){
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
if(map){
row['H'] = map.avg_price
row['I'] = map.calc_type
row['V'] = map.avg_price
// row['I'] = map.calc_type
}
}
}
@@ -33,17 +33,31 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\¶Ô±ê²¹³äÊý¾Ý.xlsx"
// addGXDPrice(filePath)
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
addThirdPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
}
def addTradePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['Y']){
def dist = CommonDatautils.findDistrictById(d['Y'])
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
if (tradePrice) {
d['W'] = tradePrice.toInteger()
}
}
});
}
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
@@ -190,11 +204,11 @@ class AddPriceTest {
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['G']) {
def dist = CommonDatautils.findDistrictById(d['G'])
if (!d['X']) {
def dist = CommonDatautils.findDistrictById(d['Y'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['Z'] = thirdPrice
d['X'] = thirdPrice
}
}
})
@@ -206,8 +220,8 @@ class AddPriceTest {
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
q.put("s_date", 202404)
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
@@ -227,12 +241,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['K']) {
def dist = CommonDatautils.findDistrictById(d['C'])
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['K'] = gxdPrice
d['M'] = gxdPrice
}
}
})
@@ -256,16 +270,16 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['W']){
if(d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(!d['L']){
if(d['B'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['W'] = dbo.avg_price
d['X'] = dbo.cal_type
// d['J'] = dbo.avg_price_2b
// d['W'] = dbo.avg_price
// d['X'] = dbo.cal_type
d['L'] = dbo.avg_price_2b
}
}
}
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
@@ -58,6 +60,9 @@ class GaoDePoiDataProcess extends BaseBatch{
def fullCityMap = null
DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL)
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
@@ -69,12 +74,215 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchMapPoi();
synchCompany()
// synchCompany()
// poiCatProcess()
// addressAndCatProcess()
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
// sysnchVillage()
addTownship()
}
def addTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d->
if(d['E']){
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
// if(gaodeMap && gaodeMap.township){
// if(gaodeMap.city && gaodeMap.city!="[]"){
// d['H'] = gaodeMap.city
// }
// if( gaodeMap.district && gaodeMap.district!='[]'){
// d['I'] = gaodeMap.district
// }
// if(gaodeMap.township && gaodeMap.township!='[]'){
// d['J'] = gaodeMap.township
// }
// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
// d['K'] = gaodeMap.format_addr
// }
//
// }
def q = new BasicDBObject()
q.put("city", d['H'])
q.put("district", d['I'])
q.put("street",d['J'])
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
if(obj){
d['L'] = "是"
}
}
},0)
}
def sysnchVillage(){
def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"]
for(def city : buducitys){
def pois = findAllVillagePoiMap(city)
def rows = createVillageData(city, pois)
save(city, rows)
saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
}
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
def townMap = findTownRange(city)
for(def i=0;i<pois.size();i++){
def p = pois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
if(!obj){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
rets.addAll(ret)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def isTown( city, region, name){
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
// obj = it
// },"192.168.0.89")
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
def body = """
{
"query": {
"bool":{
"must":[
{"term": {"city.keyword":"${city}"}},
{"term": {"district.keyword":"${region}"}},
{"term": {"town.keyword":"${town}"}}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
def obj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
obj = hits[0]
}
}
println(response)
return obj
}
def createVillageData(province, map, townMap){
// def fjw_province = province
// def fjw_city = RegionData.formatCity(map.city)
// def fjw_region = RegionData.formatRegion(fjw_city, map.district)
def retMap = [:]
retMap.province = ProvinceData.getFullProvince(province)
// retMap.fjw_province = fjw_province
// retMap.fjw_city = fjw_city
retMap.city = map.city
retMap.district = map.district
retMap.name = map.name
retMap.type = "地名地址信息"
retMap.secondary_type = "普通地名"
retMap.small_type = "村庄级地名"
retMap.cat = "地名地址信息;普通地名;村庄级地名"
retMap.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
retMap.township = InsidePolygon.isInArea2(retMap.location.lon, retMap.location.lat, townMap)
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(retMap.location.lat, retMap.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
retMap.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(retMap.location.lat, retMap.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
retMap.gps_location = gpslocation
}
if(!retMap.township){
def gaodeMap = getTownshipInfo(retMap.location.lon+","+retMap.location.lat)
if(gaodeMap && gaodeMap.township){
retMap.township = gaodeMap.township
}
// if(!fjw_region){
// if(gaodeMap && gaodeMap.district){
// fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
// }
// }
//
// if(!fjw_region){
// println fjw_city+" "+retMap.region+" 无法格式化 "+retMap.location + " "+gaodeMap
// }
// retMap.fjw_district = fjw_region
}
// retMap.fjw_district = fjw_region
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(retMap.name, " ")
retMap.version = version
retMap.data_source = "百度地图"
retMap.data_id = map._id.toString()
retMap.id = map._id.toString()
if(retMap.secondary_type && TAGS.contains(retMap.secondary_type)){
retMap.status = 0
}else{
retMap.status = 1
}
return retMap
}
def findAllVillagePoiMap(city){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
keys.put("addr", 1)
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄").append("city", city), keys).toList()
return list;
}
def sysnchPoi(){
@@ -158,7 +366,6 @@ class GaoDePoiDataProcess extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(results.size())
def prov = ProvinceData.getProvince(city.replace("ÊÐ",""))
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
@@ -348,7 +555,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def client = ElasticUtils.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
@@ -409,6 +616,8 @@ class GaoDePoiDataProcess extends BaseBatch{
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
@@ -420,6 +629,9 @@ class GaoDePoiDataProcess extends BaseBatch{
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
city = addressComponent.city
format_addr = jsonObject.regeocode.formatted_address
}
}
}
@@ -433,8 +645,10 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
@@ -58,15 +60,16 @@ class SeaweedDataSynch extends BaseBatch{
public void process(Object args) {
// run()
run()
// println "全国district, shop, office同步完成"
//民政部编码上传
codeRun()
// codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
// addTest()
// updateTownship()
}
def addTest(){
@@ -375,4 +378,16 @@ class SeaweedDataSynch extends BaseBatch{
}
}
def updateTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
if(row['Q'] && row['Q']=='是'){
def udata = new BasicDBObject()
udata.put("township", row['J'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
}
})
}
}
@@ -98,6 +98,23 @@ class InsidePolygon {
}
return list;
}
public static List<List<Point>> createPoint4(def pointList){
List<List<Point>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point> list = new ArrayList<>();
for(def arr : pointArr){
Point point = new Point();
double x = arr[0];
double y = arr[1];
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -76,7 +76,6 @@ class ElasticUtils {
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@@ -101,6 +100,30 @@ class ElasticUtils {
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 5
def nThread = Runtime.getRuntime().availableProcessors() * 4
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
/**
* 冪帠僅蛌華硊<br>
@@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery;
import com.fangjia.dal.SortParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank;
import com.fangjia.dal.mongo.model.AliasFang;
import com.fangjia.dal.mongo.model.DistrictFacilitys;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.service.bo.DistrictInfo;
@@ -540,10 +541,10 @@ public interface DistrictService extends ModeService<DistrictKey, DistrictBo> {
*
* @param city
* @param region
* @param masterDistrictNname
* @param masterDistrictName
* @return
*/
List<District> findSlaves(String city, String region, String masterDistrictNname);
List<District> findSlaves(String city, String region, String masterDistrictName);
/**
* 分期操作时设置此小区为人工操作标识
@@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService {
}
@Override
public List<District> findSlaves(String city, String region, String masterDistrictNname) {
public List<District> findSlaves(String city, String region, String masterDistrictName) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("visible", 0));
params.add(new QueryParam("city", city));
params.add(new QueryParam("region", region));
params.add(new QueryParam("masterDistrictNname", masterDistrictNname));
params.add(new QueryParam("masterDistrictName", masterDistrictName));
return districtDao.findAllByParams(params);
}
@@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService {
params.add(new QueryParam("$sr", new SrQuery(query)));
return districtDao.findAllByParams(params);
}
}
@@ -0,0 +1,23 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.model.AliasFang;
import java.util.List;
import java.util.Set;
/**
* @author shenjian
* @since 2024/6/11
*/
public interface AliasFangService {
/**
* 查询搜房别名与名称关系
*
* @param city
* @param region
* @param names
* @return
*/
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
}
@@ -0,0 +1,27 @@
package com.fangjia.service.open;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.mongo.dao.AliasFangDao;
import com.fangjia.dal.mongo.model.AliasFang;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* @author shenjian
* @since 2024/6/11
*/
public class AliasFangServiceImpl implements AliasFangService{
private AliasFangDao aliasFangDao;
@Override
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
return aliasFangDao.findFangByNameAndAlias(city, region, names);
}
public void setAliasFangDao(AliasFangDao aliasFangDao) {
this.aliasFangDao = aliasFangDao;
}
}