Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/6/11
|
||||
*/
|
||||
public interface AliasFangDao extends MongoDao<AliasFang> {
|
||||
|
||||
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
import com.fangjia.dal.mongo.model.ConsumerAddress;
|
||||
import com.mongodb.BasicDBList;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.sothis.core.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class AliasFangDaoImpl extends MongoDaoImpl<AliasFang> implements AliasFangDao{
|
||||
|
||||
@Override
|
||||
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
|
||||
DBObject query = new BasicDBObject();
|
||||
if (StringUtils.isNotBlank(city)) {
|
||||
query.put("city", city);
|
||||
}
|
||||
if (StringUtils.isNotBlank(region)) {
|
||||
query.put("region", region);
|
||||
}
|
||||
// db.alias_fang.find({'city': 'ÖØÇì', 'region': 'Óå±±', $or: [{'name': 'ÖйúĦ'}, {'alias': 'ÖйúĦ'}]})
|
||||
if(CollectionUtils.isNotEmpty(names)) {
|
||||
BasicDBList dbList = new BasicDBList();
|
||||
dbList.add(new BasicDBObject("name", new BasicDBObject("$in", names)));
|
||||
dbList.add(new BasicDBObject("alias", new BasicDBObject("$in", names)));
|
||||
query.put("$or", dbList);
|
||||
}
|
||||
List<AliasFang> objs = find(query, null, null, 0, Integer.MAX_VALUE);
|
||||
return objs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package com.fangjia.dal.mongo.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
/**
|
||||
* ËÑ·¿Ãû³ÆÓë±ðÃû¹ØÏµ
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "business.alias_fang")
|
||||
public class AliasFang implements MongoEntity {
|
||||
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private String alias;
|
||||
|
||||
@Override
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id", unique = true, nullable = false)
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "region")
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Column(name = "alias")
|
||||
public String getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
}
|
||||
@@ -187,7 +187,7 @@ public class District implements SeaweedEntity {
|
||||
private double score;
|
||||
// 小区性质
|
||||
private String landProperty;
|
||||
private String masterDistrictNname;
|
||||
private String masterDistrictName;
|
||||
private String ip;
|
||||
private String addrType;
|
||||
private boolean selected;
|
||||
@@ -309,12 +309,12 @@ public class District implements SeaweedEntity {
|
||||
}
|
||||
|
||||
@Column(name = "master_district_name")
|
||||
public String getMasterDistrictNname() {
|
||||
return masterDistrictNname;
|
||||
public String getMasterDistrictName() {
|
||||
return masterDistrictName;
|
||||
}
|
||||
|
||||
public void setMasterDistrictNname(String masterDistrictNname) {
|
||||
this.masterDistrictNname = masterDistrictNname;
|
||||
public void setMasterDistrictName(String masterDistrictName) {
|
||||
this.masterDistrictName = masterDistrictName;
|
||||
}
|
||||
|
||||
public void setCategory(String category) {
|
||||
|
||||
+96
-6
@@ -391,13 +391,106 @@ public class AddrSvcController {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则-南京银行
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object njcbVillageIdentifyAction(AddressModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
|
||||
String address = model.getAddress();
|
||||
params.put("address", address);
|
||||
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
|
||||
params.put("type", type);
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("msgCode");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if (json.containsKey("result")) {
|
||||
//我们的结果
|
||||
JSONObject result = json.getJSONObject("result");
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
|
||||
//结果比较,如果涉农结果不一致
|
||||
//调用齐梁 type :1 对公 0 对私
|
||||
//我们 type :1 对私 2 对共
|
||||
String qltype = "2".equals(type) ? "1" : "0";
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
//城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则-基础版本
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object villageIdentifyAction(AddressModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
@@ -435,11 +528,8 @@ public class AddrSvcController {
|
||||
if (json.containsKey("result")) {
|
||||
//我们的结果
|
||||
JSONObject result = json.getJSONObject("result");
|
||||
// 如果是无效信息,则用我们的
|
||||
if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) {
|
||||
basicModel.setResult(result);
|
||||
}//我们省级及以上有开发区,直接用我们的结果
|
||||
else if (result.containsKey("devDistrictName")) {
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else {
|
||||
|
||||
+40
-9
@@ -233,10 +233,13 @@ public class ComprehensiveQueryController {
|
||||
communityAvgPriceParams.put("city", model.getCity());
|
||||
communityAvgPriceParams.put("address", model.getAddress());
|
||||
communityAvgPriceParams.put("category", model.getCategory());
|
||||
communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
|
||||
// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
|
||||
communityAvgPriceParams.put("suggestName", String.valueOf(model.getName()));
|
||||
communityAvgPriceParams.put("preciseLevel", "strict");
|
||||
communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB");
|
||||
JSONObject matchResponse = null;
|
||||
try {
|
||||
matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
|
||||
matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8");
|
||||
// matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
|
||||
} catch (HttpRequestException e) {
|
||||
|
||||
@@ -256,9 +259,21 @@ public class ComprehensiveQueryController {
|
||||
DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district));
|
||||
if (null != statBo) {
|
||||
if (statBo.getDistrictStat() != null) {
|
||||
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
|
||||
// 有2B价格,使用2B价格
|
||||
Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b();
|
||||
if (avgPrice2b != null && avgPrice2b > 0) {
|
||||
dto.setAvgPrice(avgPrice2b);
|
||||
} else {
|
||||
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
|
||||
}
|
||||
} else if (statBo.getVillaDistrictStat() != null) {
|
||||
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
|
||||
// 有2B价格,使用2B价格
|
||||
Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b();
|
||||
if (avgPrice2b != null && avgPrice2b > 0) {
|
||||
dto.setAvgPrice(avgPrice2b);
|
||||
} else {
|
||||
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -638,10 +653,18 @@ public class ComprehensiveQueryController {
|
||||
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||
int factorPrice = evaluation.getAvgPrice();
|
||||
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||
Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95);
|
||||
Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
|
||||
// 有2B价格用2B价格,没有用2C
|
||||
Integer minPrice = 0;
|
||||
Integer maxPrice = 0;
|
||||
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) {
|
||||
minPrice = (int) (evaluation.getAvgPrice2b() * 0.95);
|
||||
maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05);
|
||||
} else {
|
||||
minPrice = (int) (evaluation.getAvgPrice() * 0.95);
|
||||
maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
|
||||
}
|
||||
// 偏离度超过5%不进行调整
|
||||
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
evaluation.setAvgPrice(avgPrice);
|
||||
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||
}
|
||||
@@ -657,7 +680,11 @@ public class ComprehensiveQueryController {
|
||||
districtPriceMap.put("district", district.getRegion());
|
||||
districtPriceMap.put("name", district.getName());
|
||||
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
|
||||
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
|
||||
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) {
|
||||
districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b());
|
||||
} else {
|
||||
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
|
||||
}
|
||||
|
||||
// 根据城市等 级设置D级可信度
|
||||
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
|
||||
@@ -882,7 +909,11 @@ public class ComprehensiveQueryController {
|
||||
otherPriceRets = new HashMap();
|
||||
}
|
||||
|
||||
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
|
||||
if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){
|
||||
otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b());
|
||||
}else {
|
||||
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
|
||||
}
|
||||
Map<String, Object> levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit());
|
||||
Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit);
|
||||
|
||||
|
||||
+3
-3
@@ -51,7 +51,7 @@ public class GdBankController {
|
||||
};
|
||||
|
||||
/**
|
||||
* 涉农识别-新规
|
||||
* 涉农识别-新规则-光大银行
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
@@ -95,8 +95,8 @@ public class GdBankController {
|
||||
} else if (("否").equals(result.get("countyLevelCity"))) {
|
||||
result.put("prefectureLevelCity", "是");
|
||||
}
|
||||
// 如果是无效信息,则用我们的
|
||||
if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) {
|
||||
// 如果是无效信息或者没有行政区划,则用我们的
|
||||
if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) {
|
||||
basicModel.setResult(result);
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
} else if (result.containsKey("devDistrictName")) {
|
||||
|
||||
+107
-29
@@ -10,17 +10,7 @@ import java.net.URLEncoder;
|
||||
import java.nio.charset.Charset;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@@ -30,6 +20,7 @@ import javax.validation.Valid;
|
||||
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.*;
|
||||
import com.fangjia.dal.mongo.dao.AliasFangDao;
|
||||
import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
@@ -39,6 +30,7 @@ import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.open.dto.*;
|
||||
import com.fangjia.open.dto.addressGovernance.StructureDto;
|
||||
import com.fangjia.open.util.CommonDtoUtils;
|
||||
import com.fangjia.service.address.AddressParserUtil;
|
||||
import com.fangjia.service.district.*;
|
||||
import com.fangjia.service.mongo.fangjia.*;
|
||||
import com.fangjia.service.open.*;
|
||||
@@ -138,7 +130,6 @@ import com.mongodb.DBObject;
|
||||
|
||||
@Authority(role = Role.USER)
|
||||
public class PropertyController {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
|
||||
private DistrictService districtService;
|
||||
private AddressParser addressParser;
|
||||
@@ -186,7 +177,7 @@ public class PropertyController {
|
||||
|
||||
private FileDao fileDao = FileDao.getInstance();
|
||||
private static final int DEFAULT_PAGE_SIZE = 20;
|
||||
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" };
|
||||
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" };
|
||||
static final int BAI_DU_GEO = 1;// 百度系坐标
|
||||
static final int GAO_DE_GEO = 2;// 高德系坐标
|
||||
|
||||
@@ -196,6 +187,7 @@ public class PropertyController {
|
||||
private DistrictRoomSqmetreRentService districtRoomSqmetreRentService;
|
||||
private RentDealPriceService rentDealPriceService;
|
||||
private EvaluateRatioService evaluateRatioService;
|
||||
private AliasFangService aliasFangService;
|
||||
|
||||
private static final int EVALUATE_SHOW_DETAIL = 1;
|
||||
private static final Map<String, String> PROPERTY_TYPES = new HashMap<String, String>() {
|
||||
@@ -609,6 +601,25 @@ public class PropertyController {
|
||||
dto.setBlock(district.getBlock());
|
||||
}
|
||||
dto.setPropertyType(district.getEstateType2());
|
||||
if (CollectionUtils.isNotEmpty(district.getAlias())) {
|
||||
// 将最相近的别名输出给用户
|
||||
String alias = null;
|
||||
double similarity = 0;
|
||||
if (StringUtils.isNotBlank(model.getKeyword())) {
|
||||
for (String mappingAlias : district.getAlias()) {
|
||||
double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword());
|
||||
if (curr > similarity ) {
|
||||
similarity = curr;
|
||||
alias = mappingAlias;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (StringUtils.isNotBlank(alias)) {
|
||||
dto.setAlias(alias);
|
||||
} else {
|
||||
dto.setAlias(district.getAlias().get(0));
|
||||
}
|
||||
}
|
||||
list.add(dto);
|
||||
}
|
||||
|
||||
@@ -2516,7 +2527,7 @@ public class PropertyController {
|
||||
model.setMax(100);
|
||||
}
|
||||
|
||||
List<String> districtNames = new ArrayList<String>();
|
||||
Set<String> districtNames = new HashSet<String>();
|
||||
if (null == district) {
|
||||
districtNames.add(model.getName());
|
||||
} else {
|
||||
@@ -2524,14 +2535,56 @@ public class PropertyController {
|
||||
if (district.getAlias() != null) {
|
||||
districtNames.addAll(district.getAlias());
|
||||
}
|
||||
}
|
||||
List<DealPrice> dealPrices = null;
|
||||
for (String name : districtNames) {
|
||||
dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
|
||||
if (!dealPrices.isEmpty()) {
|
||||
break;
|
||||
final String masterDistrictName = district.getMasterDistrictName();
|
||||
if (StringUtils.isNotBlank(masterDistrictName)) {
|
||||
List<District> districts;
|
||||
if ("master".equals(masterDistrictName)) {
|
||||
// 所有的分期数据
|
||||
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName());
|
||||
} else {
|
||||
// 根据主小区名查询所有名称
|
||||
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(districts)) {
|
||||
for (District districtTemp : districts) {
|
||||
districtNames.add(districtTemp.getName());
|
||||
}
|
||||
}
|
||||
|
||||
// districtNames 不包含 主小区名,加进去
|
||||
if (!districtNames.contains(masterDistrictName)){
|
||||
districtNames.add(masterDistrictName);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
// 将搜房中的别名与名称加入查询集合
|
||||
try {
|
||||
List<AliasFang> aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames);
|
||||
Set<String> aliasSet = new HashSet<String>();
|
||||
for (AliasFang aliasFang : aliasFangs) {
|
||||
if (StringUtils.isNotBlank(aliasFang.getName())) {
|
||||
aliasSet.add(aliasFang.getName());
|
||||
}
|
||||
if (StringUtils.isNotBlank(aliasFang.getAlias())) {
|
||||
aliasSet.add(aliasFang.getAlias());
|
||||
}
|
||||
}
|
||||
districtNames.addAll(aliasSet);
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
// List<DealPrice> dealPrices = new ArrayList<DealPrice>();
|
||||
// 将主名与映射别名查得成交加入 存在不能查出最新成交问题
|
||||
// for (String name : districtNames) {
|
||||
// List<DealPrice> dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
|
||||
// if (CollectionUtils.isNotEmpty(dealPricesTempList)) {
|
||||
// dealPrices.addAll(dealPricesTempList);
|
||||
// }
|
||||
// }
|
||||
List<DealPrice> dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList<String>(districtNames), model.getMax() * 5, model);
|
||||
if (model.getSize() != null) {
|
||||
Collections.sort(dealPrices, new Comparator<DealPrice>() {
|
||||
@Override
|
||||
@@ -2580,15 +2633,9 @@ public class PropertyController {
|
||||
}
|
||||
deal = new LinkedHashMap<String, Object>();
|
||||
dealPrice.setId(null);
|
||||
if (district != null) {
|
||||
deal.put("city", district.getCity());
|
||||
deal.put("district", district.getRegion());
|
||||
deal.put("name", district.getName());
|
||||
} else {
|
||||
deal.put("city", dealPrice.getCity());
|
||||
deal.put("district", dealPrice.getRegion());
|
||||
deal.put("name", dealPrice.getDistrictName());
|
||||
}
|
||||
deal.put("city", dealPrice.getCity());
|
||||
deal.put("district", dealPrice.getRegion());
|
||||
deal.put("name", dealPrice.getDistrictName());
|
||||
if (dealPrice.getArea() == null) {
|
||||
deal.put("size", "");
|
||||
} else {
|
||||
@@ -3501,6 +3548,33 @@ public class PropertyController {
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<DealPrice> findDealPricesByNames(String city, String region, List<String> names, int max,
|
||||
DealPriceModel model) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("city", city));
|
||||
params.add(new QueryParam("region", region));
|
||||
params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names));
|
||||
if (model.getRoom() != null) {
|
||||
params.add(new QueryParam("room", model.getRoom()));
|
||||
}
|
||||
if (model.getHall() != null) {
|
||||
params.add(new QueryParam("hall", model.getHall()));
|
||||
}
|
||||
// 18年之后抓住的数据没有score字段
|
||||
// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l));
|
||||
List<DealPrice> dealPrices = dealPriceService.findDealPrices(params, max);
|
||||
List<DealPrice> distinctDealPrices = new ArrayList<DealPrice>();
|
||||
for (DealPrice dealPrice : dealPrices) {
|
||||
if (dealPrice.getAvgPrice() == null) {
|
||||
dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea()));
|
||||
}
|
||||
if (!checkExists(distinctDealPrices, dealPrice)) {
|
||||
distinctDealPrices.add(dealPrice);
|
||||
}
|
||||
}
|
||||
return distinctDealPrices;
|
||||
}
|
||||
|
||||
private List<DealPrice> findDealPricesByName(String city, String region, String name, int max,
|
||||
DealPriceModel model) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
@@ -5959,4 +6033,8 @@ public class PropertyController {
|
||||
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
|
||||
this.communityBasePriceService = communityBasePriceService;
|
||||
}
|
||||
|
||||
public void setAliasFangService(AliasFangService aliasFangService) {
|
||||
this.aliasFangService = aliasFangService;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,10 @@ public class PropertyDto {
|
||||
private String propertyType;
|
||||
private String block;
|
||||
private String distance;
|
||||
/**
|
||||
* 新增小区别名
|
||||
*/
|
||||
private String alias;
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
@@ -88,4 +92,12 @@ public class PropertyDto {
|
||||
public void setDistance(String distance) {
|
||||
this.distance = distance;
|
||||
}
|
||||
|
||||
public String getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
}
|
||||
|
||||
+76
-52
@@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -61,61 +62,67 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
println objectId
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
//
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
// println "历史id加载量:"+dealIds.size()
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
//// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
//
|
||||
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
//// saveToPgSql2(datas)
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
// delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
// saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
@@ -682,7 +689,13 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def updateCrawlTradeDate(){
|
||||
|
||||
def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20"
|
||||
def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2)
|
||||
def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
println(objectId)
|
||||
CcbCommonUtils.updateTrades(startDate, endDate, objectId)
|
||||
}
|
||||
|
||||
|
||||
@@ -822,4 +835,15 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def dateToObjectId(Date date) {
|
||||
// 获取时间戳(单位:秒)
|
||||
def timestamp = (date.time / 1000) as int
|
||||
def hexTimestamp = Integer.toHexString(timestamp)
|
||||
|
||||
// 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值
|
||||
def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000'
|
||||
return new ObjectId(objectIdStr)
|
||||
}
|
||||
}
|
||||
|
||||
+17
-4
@@ -15,9 +15,10 @@ import groovy.sql.Sql
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
//
|
||||
public static def citys_fjw_all = [ "乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
|
||||
public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
|
||||
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
|
||||
|
||||
@@ -439,7 +440,7 @@ class CcbCommonUtils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","新泾三村")
|
||||
// q.put("name","鸿运大院")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
@@ -711,4 +712,16 @@ class CcbCommonUtils {
|
||||
return map
|
||||
}
|
||||
|
||||
public static def updateTrades(startDate, endDate, cDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("_id", new BasicDBObject('$gte', cDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
if(count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("trade_date", endDate)
|
||||
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -389,7 +389,7 @@ class DealPriceDataProcess {
|
||||
def deleteThirdPartyData(){
|
||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+29
-5
@@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['J']){
|
||||
@@ -1301,20 +1301,44 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['A']){
|
||||
if(!d['A'] && d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
d['K'] = dist.estate_type2
|
||||
d['L'] = dist.status
|
||||
d['M'] = dist.name_cat
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
|
||||
// d['H'] = dist.name
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
// if(d['B']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(obj){
|
||||
// def markMap = obj.mark_map
|
||||
// if(!obj.mark_map){
|
||||
// markMap = [:]
|
||||
// }
|
||||
// def data = new BasicDBObject()
|
||||
// markMap.put("exclusive", "是")
|
||||
// data.put("mark_map", markMap)
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// if(d['B'] && !d['D']){
|
||||
|
||||
+1
-1
@@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
// }
|
||||
// }
|
||||
//
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
|
||||
+110
-39
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def size = regionDatas.size()
|
||||
final CountDownLatch latch = new CountDownLatch(size)
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
for(def i = 0;i<regionDatas.size();i++){
|
||||
final def arr = regionDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
totalCount.addAndGet(1)
|
||||
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
|
||||
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
|
||||
if(row){
|
||||
rets << row
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// };
|
||||
// })
|
||||
}
|
||||
latch.await()
|
||||
// latch.await()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
|
||||
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def allDistRegionPreMap = [:]
|
||||
def allLastMonthMap = [:]
|
||||
def allLastYearMap = [:]
|
||||
def allCcbPriceMap = [:]
|
||||
def allCcbPriceCaseCountMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
|
||||
allDistricts.addAll(districts)
|
||||
|
||||
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
allPriceMap = allPriceMap + priceMap
|
||||
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
// allPriceMap = allPriceMap + priceMap
|
||||
|
||||
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
|
||||
allRentMap = allRentMap + rentMap
|
||||
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
|
||||
allDealCountMap = allDealCountMap + dealCountMap
|
||||
|
||||
|
||||
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
|
||||
|
||||
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceMap = allCcbPriceMap + datasMap
|
||||
|
||||
|
||||
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
|
||||
|
||||
def cityRankMap = findCommunityCityRank(datas)
|
||||
allCityRankMap = allCityRankMap + cityRankMap
|
||||
|
||||
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
allLastYearMap = allLastYearMap + lastYearMonthMap
|
||||
}
|
||||
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(cityLists){
|
||||
rets.addAll(cityLists)
|
||||
}
|
||||
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def totalCount = new AtomicLong()
|
||||
def distCount = districts.size()
|
||||
final CountDownLatch latch = new CountDownLatch(distCount)
|
||||
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(row){
|
||||
rets << row
|
||||
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
|
||||
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def sellDataType = 1
|
||||
def rentDataType = 1
|
||||
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
//房源查询是否按建行均价筛选房源
|
||||
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
def sell_avg_price = null
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
def sell_count = 0
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
|
||||
|
||||
if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
|
||||
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
}
|
||||
}
|
||||
if(sell_count){
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(sellMap){
|
||||
sell_count = sellMap.avg_price.size()
|
||||
if(sell_count>3){
|
||||
sell_avg_price = avgValue(sellMap.avg_price)
|
||||
if(sellMap){
|
||||
sell_avg_area = avgValue(sellMap.area)
|
||||
sell_avg_total = avgValue(sellMap.total_price)
|
||||
}else{
|
||||
sell_count = 0
|
||||
sellDataType = null
|
||||
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// if(sell_count>3){
|
||||
// if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def rent_avg_price = null
|
||||
def rent_avg_area = null
|
||||
def rent_avg_total = null
|
||||
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
rentDataType = null
|
||||
}
|
||||
}
|
||||
if(!sell_avg_price){
|
||||
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
|
||||
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
|
||||
// if(!sell_avg_price){
|
||||
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
// }
|
||||
if(!rent_avg_price){
|
||||
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
|
||||
def avgPrice = null
|
||||
if(region){
|
||||
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
|
||||
}else{
|
||||
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
|
||||
}
|
||||
|
||||
|
||||
if (sell_count < 10) {
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
sellStartDate = startDate
|
||||
}
|
||||
if (sell_count) {
|
||||
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
|
||||
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
|
||||
}
|
||||
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return datas;
|
||||
}
|
||||
|
||||
def findCommunityPriceByCityMap(cityCode, month){
|
||||
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCommunityPriceCountByCityMap(cityCode, month){
|
||||
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
if(r.gujiabdsrt3mlgcasenum){
|
||||
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
|
||||
}
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCcbCityPrice(cityCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def avgPrice = null
|
||||
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return regionPriceMap;
|
||||
}
|
||||
|
||||
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
|
||||
def price = null
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
price = r.gujia_bds_clc_avprc
|
||||
})
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
def districtCityPrictRate(cityPrice, datas){
|
||||
def distCityRateMap = [:]
|
||||
|
||||
+177
-2
@@ -13,11 +13,13 @@ import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
@@ -117,6 +119,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -274,7 +277,7 @@ class CommonDatautils {
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","揭흩소蹈")
|
||||
// q.put("name","托斯卡纳庄园")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -283,7 +286,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -4571,6 +4574,28 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def findAllTownRanges(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("town", 1)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def lists = townMap.get(d.city)
|
||||
if(!lists){
|
||||
lists = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.region
|
||||
t.town = d.town
|
||||
lists << t
|
||||
townMap.put(d.city, lists)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
@@ -4605,6 +4630,108 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryFullNameByCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","城市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = RegionData.formatCity(d.city)
|
||||
if(city){
|
||||
cityMap.put(city, d)
|
||||
}else{
|
||||
//println d.city +" 格式化不出"
|
||||
}
|
||||
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
static def queryDistNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.mzb_district_name){
|
||||
def regions = regionMap.get(d.city)
|
||||
if(!regions){
|
||||
regions = []
|
||||
}
|
||||
regions << d
|
||||
regionMap.put(d.city, regions)
|
||||
}else{
|
||||
println d.province+" "+d.district +" 格式化不出民政部区域"
|
||||
}
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryCommunityByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.district
|
||||
t.town = d.street
|
||||
list << t
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -4675,6 +4802,54 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+257
-71
@@ -3,7 +3,9 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -24,13 +26,14 @@ class DevProcess {
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
// task.devSplitRoad()
|
||||
task.devNameSplitRoad()
|
||||
task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
|
||||
@@ -44,7 +47,15 @@ class DevProcess {
|
||||
// task.addNewDev()
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
// task.addDevShape()//添加开发区围栏
|
||||
// task.matchDevTownship()//添加乡镇街道
|
||||
|
||||
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
|
||||
}
|
||||
|
||||
|
||||
@@ -162,69 +173,101 @@ class DevProcess {
|
||||
}
|
||||
}
|
||||
|
||||
def findDev(city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
q.put("city", city)
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
def i = 10;
|
||||
while(i<11){
|
||||
def fileName = "/home/goojia/gj/company-${i}.xlsx"
|
||||
def i = 0;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\company-${i}.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx"
|
||||
}
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
/*if(d['C'] && !d['V']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['V'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['W'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['X'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['Y'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['Z'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
}*/
|
||||
if(d['C'] && d['B']!='河北省'){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C'])
|
||||
if(jsonObject){
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['E'] && !d['G']){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||
def devName = jsonObject1.get("dev")
|
||||
def obj = findDev(d['B'], devName)
|
||||
d['F'] = devName
|
||||
if (obj) {
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
|
||||
d['O'] = jsonObject.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
i++
|
||||
if(d['C'] && !d['S']) {
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
d['G'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
|
||||
d['H'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
|
||||
d['I'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
|
||||
d['J'] = jsonObject.get("township")
|
||||
}
|
||||
|
||||
if (jsonObject && street && streetNumber && street != "null") {
|
||||
d['K'] = jsonObject.get("street")
|
||||
def road = jsonObject.get("street")
|
||||
if (streetNumber != 'null') {
|
||||
d['L'] = jsonObject.get("streetNumber")
|
||||
road = road + jsonObject.get("streetNumber")
|
||||
} else {
|
||||
d['N'] = "信息不完整"
|
||||
}
|
||||
|
||||
if (!d['N'] && !d['C'].contains(road)) {
|
||||
d['N'] = "原地址不含路"
|
||||
}
|
||||
|
||||
if (!d['N']) {
|
||||
def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$')
|
||||
if (!flag) {
|
||||
d['N'] = "路非法"
|
||||
}
|
||||
def roadsize = jsonObject.get("street").size()
|
||||
if (!d['N'] && (roadsize < 3 || roadsize > 7)) {
|
||||
d['N'] = "路太长|路太短"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['N'] = "无地址"
|
||||
}
|
||||
|
||||
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
|
||||
def devName = jsonObject.get("dev")
|
||||
def obj = findDev(d['B'], devName)
|
||||
d['M'] = devName
|
||||
if (obj) {
|
||||
d['S'] = obj._id.toString()
|
||||
d['T'] = obj.name
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -234,7 +277,7 @@ class DevProcess {
|
||||
def devNameSplitRoad(){
|
||||
def fileName = "/home/goojia/gj/company.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx"
|
||||
}
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['P'] && d['P']=="·Õý³£" && !d['O']){
|
||||
@@ -291,17 +334,18 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
|
||||
if(d['H']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
|
||||
if(!d['Q'] && d['L']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['H'])
|
||||
q.put("name", d['L'])
|
||||
q.put("city",d['G'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['S'] = obj._id.toString()
|
||||
d['T'] = obj.name
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
}
|
||||
}
|
||||
},1);
|
||||
},0);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
@@ -506,6 +550,27 @@ class DevProcess {
|
||||
})
|
||||
}
|
||||
|
||||
def addNewDev2(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
|
||||
if(d['F'] && d['F']=="新增"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.create_time = new Date()
|
||||
udata.province = d['G']
|
||||
udata.city = d['I']
|
||||
udata.district = d['J']
|
||||
udata.level = "省级"
|
||||
udata.name = d['D']
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
|
||||
if(!obj){
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
println udata.name +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createDevMapping(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
for(def d : dbcursor){
|
||||
@@ -548,6 +613,26 @@ class DevProcess {
|
||||
});
|
||||
}
|
||||
|
||||
def deleteAddr2(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d->
|
||||
if(d['Q'] && d['M']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['Q']))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def q2 = new BasicDBObject()
|
||||
q2.put("name", obj.name)
|
||||
q2.put("address", d['I']+d['J'])
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q2)
|
||||
if(sameObj){
|
||||
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def synchDevTo43(){
|
||||
def dbcursor = development_zone_CONN.find()
|
||||
def rets = []
|
||||
@@ -559,17 +644,19 @@ class DevProcess {
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx",{d->
|
||||
if(d['P'] && d['P']=="路正常" && d['O']){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
|
||||
if(d['R']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['O'])
|
||||
q.put("name", d['R'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
def address = d['M']+d['N']
|
||||
d['P'] = obj._id.toString()
|
||||
d['Q'] = obj.name
|
||||
def address = d['L']+d['M']
|
||||
q.put("address", address)
|
||||
// def street = d['G']
|
||||
// q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -583,11 +670,110 @@ class DevProcess {
|
||||
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "µØÖ·")
|
||||
udata.put("address", address)
|
||||
udata.put("address",address)
|
||||
// udata.put("type", "乡镇街道")
|
||||
// udata.put("township", street)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+address
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchDevTownship(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
|
||||
if(d['C']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['C'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = obj._id.toString()
|
||||
def street = d['D']
|
||||
q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
if(obj.city){
|
||||
udata.put("city", obj.city)
|
||||
}
|
||||
if(obj.district){
|
||||
udata.put("district", obj.district)
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
// udata.put("type", "乡镇街道")
|
||||
// udata.put("township", street)
|
||||
|
||||
udata.put("type", "区域")
|
||||
udata.put("dev_district", street)
|
||||
udata.put("c_date",new Date())
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+street
|
||||
}
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def addAlias(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['D'] = obj._id.toString()
|
||||
def arrs = d['B'].split("、")
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevShape(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d->
|
||||
if(d['W']){
|
||||
def s = d['S']
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
Map<String, Object> map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", map)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevStreet(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
|
||||
if(d['J']=="是"){
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道"))
|
||||
for(def obj : dbcursor){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province" , obj.province)
|
||||
o.put("city" , obj.city)
|
||||
o.put("district" , obj.mzb_district_name)
|
||||
o.put("name" , d['E'])
|
||||
o.put("type" , "乡镇街道")
|
||||
o.put("township" , obj.street)
|
||||
DEV_MAPPING_CONN.save(o)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
|
||||
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
|
||||
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("run_status", new BasicDBObject('$exists', false))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>=10000){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
updateToEs2(datas)
|
||||
updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = running(d)
|
||||
rets << map
|
||||
if(map.update_status && map.update_status!=100){
|
||||
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def regions = getFullRegion(d)
|
||||
def town = null
|
||||
def map =[:]
|
||||
if(d.location){
|
||||
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if(d.district){
|
||||
for(def rs : regions){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
|
||||
if(regionFlag){
|
||||
//判断township是否在统计局中
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, d.city, d.township)
|
||||
}
|
||||
if(town){
|
||||
if(town.street == d.township){
|
||||
//打标记
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = town.street
|
||||
}
|
||||
}else{
|
||||
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
|
||||
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
|
||||
if (mapTown) {
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = mapTown.street
|
||||
}else{
|
||||
if(d.district!=gaodeMap.district){
|
||||
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
//打标记“经纬度有误,不在区域围栏中”
|
||||
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.district)){
|
||||
names << r
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def getSimilarTown(regions, city, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(city+"#"+r.district)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.township='${township}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}
|
||||
|
||||
|
||||
def updateToEs2(rows) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.update_status && map.update_status==2){
|
||||
map.remove("run_status")
|
||||
map.remove("update_status")
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def updateToMG(rows){
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def udata = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", m.id)
|
||||
if(udata){
|
||||
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println m.toString()+" 无法修改-无值"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+130
-1
@@ -50,7 +50,7 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
synchTown()
|
||||
// synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
@@ -61,6 +61,13 @@ class TownMiddleProcess {
|
||||
|
||||
// addDistrictDiffFlag()
|
||||
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -800,4 +807,126 @@ class TownMiddleProcess {
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
});
|
||||
}
|
||||
|
||||
def updateCommunityDiffStreet(){
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city_name", d.city)
|
||||
query.put("area_name", d.district)
|
||||
query.put("street_name", d.street)
|
||||
def obj = COMMUNITY_CONN.findOne(query)
|
||||
if(!obj){
|
||||
query.put("street_name", d.district+d.street)
|
||||
obj = COMMUNITY_CONN.findOne(query)
|
||||
if(obj){
|
||||
def newStreet = obj.street_name.replace(d.district, "")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_name", newStreet)
|
||||
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
|
||||
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
|
||||
}else{
|
||||
println d.city +" "+d.district+" "+d.street
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updateTownPoline(){
|
||||
def city = "唐山市"
|
||||
def name = "高新技术开发区街道"
|
||||
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
|
||||
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("city", city).append("street",name)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def checkTown(){
|
||||
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def ad : dbcursor){
|
||||
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
|
||||
def count = TOWNPOLYLINE_COLL.count(polyline_q)
|
||||
if(count==1){
|
||||
println ad.city+" "+ ad.street+" "+count
|
||||
|
||||
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("polyline_source", "区域+街道匹配")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", ad._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateTownshipByRegion(){
|
||||
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("mzb_district_name",1)
|
||||
keys.put("mzb_district_code",1)
|
||||
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
|
||||
for(def d : dbcursor){
|
||||
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mzb_district_name",d.mzb_district_name)
|
||||
udata.put("mzb_district_code",d.mzb_district_code)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
def fullCityMap = null
|
||||
def regionMap = null
|
||||
def commMap = null
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
task.process()
|
||||
|
||||
}
|
||||
|
||||
|
||||
void process() {
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
if(["中山"].contains(city)){
|
||||
continue
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkData(def dists){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << running(d)
|
||||
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return dists
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
def oldSimlarTown = null
|
||||
def newSimlarTown = null
|
||||
if(d.lng){
|
||||
d.lng = d.lng/10000000000.0
|
||||
d.lat = d.lat/10000000000.0
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, fullCity, d.township)
|
||||
}
|
||||
|
||||
if(town && town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
d.polygon_town = town.street
|
||||
}
|
||||
}
|
||||
d.strand_street = ""
|
||||
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
|
||||
if(mapTown){
|
||||
d.strand_street = mapTown.street
|
||||
if(mapTown.shape && mapTown.shape.coordinates){
|
||||
d.map_town_shape = "有"
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
|
||||
if(flag){
|
||||
d.map_polygon_town = mapTown.street
|
||||
}
|
||||
}
|
||||
}else{
|
||||
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
|
||||
if(oldSimlarTown){
|
||||
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
|
||||
d.old_similar_town_score = oldSimlarTown.score
|
||||
}
|
||||
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
|
||||
if(newSimlarTown){
|
||||
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
|
||||
d.new_similar_town_score = newSimlarTown.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def mapCity = null
|
||||
def mapRegion = null
|
||||
def mapTownship = null
|
||||
def mapAddr = null
|
||||
if(gaodeMap){
|
||||
if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
mapCity = gaodeMap.city
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
}
|
||||
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
mapAddr = gaodeMap.format_addr
|
||||
}
|
||||
}
|
||||
d.mapCity = mapCity
|
||||
d.mapRegion = mapRegion
|
||||
d.mapTownship = mapTownship
|
||||
d.mapAddr = mapAddr
|
||||
if(mapCity && mapRegion){
|
||||
if(mapCity == fullCity.city){
|
||||
if(regions.contains(mapRegion)){
|
||||
if(mapTownship){
|
||||
if(town){
|
||||
d.old_tjj_township = "是"
|
||||
}else{
|
||||
d.old_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTown){
|
||||
d.new_tjj_township = "是"
|
||||
}else{
|
||||
d.new_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTownship == d.township){
|
||||
if(town){
|
||||
d.remark = "相同且为统计局乡镇"
|
||||
}else{
|
||||
d.remark = "非统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}else{
|
||||
if(mapTown){
|
||||
d.remark = "更正"
|
||||
}else{
|
||||
d.remark = "不同且不为统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d.remark = "地图无乡镇街道"
|
||||
d.opt = "地图无街道信息"
|
||||
}
|
||||
|
||||
}else{
|
||||
d.remark = "区域不同"
|
||||
d.opt = "经纬度可能跨区域"
|
||||
}
|
||||
}else{
|
||||
d.remark = "城市不同"
|
||||
d.opt = "经纬度可能跨城市"
|
||||
}
|
||||
}
|
||||
|
||||
if(d.map_town_shape && d.map_town_shape=="有"){
|
||||
if(!d.map_polygon_town){
|
||||
d.opt = "经纬度可疑"
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def getSimilarTownByStr(fullCity, townstr, townList){
|
||||
|
||||
def maxScore = 0
|
||||
def town = null
|
||||
for(def townObj : townList){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
|
||||
if(score > maxScore){
|
||||
maxScore = score
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
town.score = maxScore
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getSimilarTown(regions, fullCity, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(fullCity.city+"#"+r)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getFullRegion(fullCity, d){
|
||||
def regions = regionMap.get(fullCity.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
|
||||
def createDatas(dists){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
|
||||
for(def d : dists){
|
||||
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -1,13 +1,46 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class TradeDatasDistBatch {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
// def
|
||||
def task = new TradeDatasDistBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
udata.put("name", d['C'])
|
||||
udata.put("alias", d['D'])
|
||||
alias_fang_COLL.save(udata)
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def flag = null
|
||||
// if(!dist){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }else{
|
||||
// flag = "名称匹配"
|
||||
// }
|
||||
// if(flag){
|
||||
// d['F'] = dist.city
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -354,7 +354,7 @@ class FangjiaClientUtils {
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+39
-8
@@ -72,7 +72,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataForsaleBatch(), args);
|
||||
@@ -86,12 +89,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19"
|
||||
def endDate = curMonth+"19" //正式运行时curMonth要改
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
@@ -133,7 +135,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
|
||||
|
||||
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
@@ -161,7 +163,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
if(!houseIdsfilter.mightContain(m.houseId)){
|
||||
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
@@ -180,6 +182,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
curhousefilter.put(m.houseId)
|
||||
houseIdsfilter.put(m.houseId)
|
||||
}else{
|
||||
println city +" "+m.houseId+" 已存在"
|
||||
}
|
||||
@@ -192,7 +196,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
@@ -268,7 +272,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
|
||||
|
||||
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
// if(rets1){
|
||||
// def newRets = []
|
||||
@@ -707,6 +711,33 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
def rets = []
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(3000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
|
||||
+6
-5
@@ -49,14 +49,15 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = ["ÎÚ³ľÆë"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
if(sDate<202004){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
}
|
||||
month = month -1
|
||||
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
if(sDate == 201912){
|
||||
|
||||
+26
-26
@@ -13,7 +13,7 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def inputfile = "D:\\ĘýžÝ\\˝¨ĐĐ\\ˇż˛úšŔÖľ\\202311šăÖÝ.xlsx"
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
@@ -69,21 +69,21 @@ class MsbPocTest {
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['S'] && row['D']){
|
||||
if(!row['Y'] && row['G']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['H']
|
||||
def region = null //row['D']
|
||||
def name = row['E']
|
||||
def address = row['G']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['S'] = jsonObject.get("id")
|
||||
row['T'] = jsonObject.get("city")
|
||||
row['U'] = jsonObject.get("district")
|
||||
row['V'] = jsonObject.get("name")
|
||||
row['W'] = jsonObject.get("propertyType")
|
||||
row['X'] = jsonObject.get("credit")
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("propertyType")
|
||||
// row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -111,18 +111,18 @@ class MsbPocTest {
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['N']){
|
||||
def city = row['A']
|
||||
def region = row['B']
|
||||
def address = row['C'] //row['G']//row['F']
|
||||
if(!row['Y']){
|
||||
def city = row['C']
|
||||
def region = null //row['B']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
if(jsonObject != null) {
|
||||
row['N'] = jsonObject.get("id")
|
||||
row['O'] = jsonObject.get("city")
|
||||
row['P'] = jsonObject.get("district")
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -169,17 +169,17 @@ class MsbPocTest {
|
||||
def addCCBPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['H']){
|
||||
if(!row['V']){
|
||||
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
// if(dist){
|
||||
if(row['D'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['D'])
|
||||
if(row['Y'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['Y'])
|
||||
if(d){
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row['H'] = map.avg_price
|
||||
row['I'] = map.calc_type
|
||||
row['V'] = map.avg_price
|
||||
// row['I'] = map.calc_type
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+31
-17
@@ -33,17 +33,31 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\¶Ô±ê²¹³äÊý¾Ý.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addTradePrice(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
if(d['Y']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
|
||||
if (tradePrice) {
|
||||
d['W'] = tradePrice.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
@@ -190,11 +204,11 @@ class AddPriceTest {
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['G']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if (!d['X']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['Z'] = thirdPrice
|
||||
d['X'] = thirdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -206,8 +220,8 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
|
||||
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
q.put("s_date", 202404)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
@@ -227,12 +241,12 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['K']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['K'] = gxdPrice
|
||||
d['M'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -256,16 +270,16 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['W']){
|
||||
if(d['G'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(!d['L']){
|
||||
if(d['B'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['W'] = dbo.avg_price
|
||||
d['X'] = dbo.cal_type
|
||||
// d['J'] = dbo.avg_price_2b
|
||||
// d['W'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
d['L'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
|
||||
'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
|
||||
+218
-4
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
@@ -58,6 +60,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
def fullCityMap = null
|
||||
|
||||
DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
@@ -69,12 +74,215 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// synchMapPoi();
|
||||
|
||||
|
||||
synchCompany()
|
||||
// synchCompany()
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
// sysnchVillage()
|
||||
|
||||
addTownship()
|
||||
}
|
||||
|
||||
def addTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d->
|
||||
if(d['E']){
|
||||
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
// d['H'] = gaodeMap.city
|
||||
// }
|
||||
// if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
// d['I'] = gaodeMap.district
|
||||
// }
|
||||
// if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
// d['J'] = gaodeMap.township
|
||||
// }
|
||||
// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
// d['K'] = gaodeMap.format_addr
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['H'])
|
||||
q.put("district", d['I'])
|
||||
q.put("street",d['J'])
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
|
||||
if(obj){
|
||||
d['L'] = "是"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def sysnchVillage(){
|
||||
def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"]
|
||||
for(def city : buducitys){
|
||||
def pois = findAllVillagePoiMap(city)
|
||||
def rows = createVillageData(city, pois)
|
||||
|
||||
save(city, rows)
|
||||
saveToMG(city, rows)
|
||||
println city +" "+pois.size()+" add:"+rows.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def createVillageData(city, pois){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
def townMap = findTownRange(city)
|
||||
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
|
||||
if(!obj){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
|
||||
def ret = createVillageData(prov, p, townMap)
|
||||
rets.addAll(ret)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def isTown( city, region, name){
|
||||
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
|
||||
// obj = it
|
||||
// },"192.168.0.89")
|
||||
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool":{
|
||||
"must":[
|
||||
{"term": {"city.keyword":"${city}"}},
|
||||
{"term": {"district.keyword":"${region}"}},
|
||||
{"term": {"town.keyword":"${town}"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
def obj = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
obj = hits[0]
|
||||
}
|
||||
}
|
||||
println(response)
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def createVillageData(province, map, townMap){
|
||||
// def fjw_province = province
|
||||
// def fjw_city = RegionData.formatCity(map.city)
|
||||
// def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
|
||||
def retMap = [:]
|
||||
retMap.province = ProvinceData.getFullProvince(province)
|
||||
// retMap.fjw_province = fjw_province
|
||||
// retMap.fjw_city = fjw_city
|
||||
|
||||
retMap.city = map.city
|
||||
retMap.district = map.district
|
||||
retMap.name = map.name
|
||||
retMap.type = "地名地址信息"
|
||||
retMap.secondary_type = "普通地名"
|
||||
retMap.small_type = "村庄级地名"
|
||||
retMap.cat = "地名地址信息;普通地名;村庄级地名"
|
||||
|
||||
|
||||
retMap.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
|
||||
retMap.township = InsidePolygon.isInArea2(retMap.location.lon, retMap.location.lat, townMap)
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(retMap.location.lat, retMap.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
retMap.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(retMap.location.lat, retMap.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
retMap.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!retMap.township){
|
||||
|
||||
def gaodeMap = getTownshipInfo(retMap.location.lon+","+retMap.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
retMap.township = gaodeMap.township
|
||||
}
|
||||
|
||||
// if(!fjw_region){
|
||||
// if(gaodeMap && gaodeMap.district){
|
||||
// fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(!fjw_region){
|
||||
// println fjw_city+" "+retMap.region+" 无法格式化 "+retMap.location + " "+gaodeMap
|
||||
// }
|
||||
|
||||
// retMap.fjw_district = fjw_region
|
||||
}
|
||||
// retMap.fjw_district = fjw_region
|
||||
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(retMap.name, " ")
|
||||
retMap.version = version
|
||||
retMap.data_source = "百度地图"
|
||||
retMap.data_id = map._id.toString()
|
||||
retMap.id = map._id.toString()
|
||||
|
||||
if(retMap.secondary_type && TAGS.contains(retMap.secondary_type)){
|
||||
retMap.status = 0
|
||||
}else{
|
||||
retMap.status = 1
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findAllVillagePoiMap(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("addr", 1)
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄").append("city", city), keys).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
def sysnchPoi(){
|
||||
@@ -158,7 +366,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def prov = ProvinceData.getProvince(city.replace("ÊÐ",""))
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -348,7 +555,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def client = ElasticUtils.getClient("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
@@ -409,6 +616,8 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
@@ -420,6 +629,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
city = addressComponent.city
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -433,8 +645,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
+17
-2
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -58,15 +60,16 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
// run()
|
||||
run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
codeRun()
|
||||
// codeRun()
|
||||
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
// addTest()
|
||||
// updateTownship()
|
||||
}
|
||||
|
||||
def addTest(){
|
||||
@@ -375,4 +378,16 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updateTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
|
||||
if(row['Q'] && row['Q']=='是'){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", row['J'])
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+17
@@ -98,6 +98,23 @@ class InsidePolygon {
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<List<Point>> createPoint4(def pointList){
|
||||
List<List<Point>> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
List<Point> list = new ArrayList<>();
|
||||
for(def arr : pointArr){
|
||||
Point point = new Point();
|
||||
double x = arr[0];
|
||||
double y = arr[1];
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个经纬度点与区域的关系
|
||||
|
||||
+24
-1
@@ -76,7 +76,6 @@ class ElasticUtils {
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@@ -101,6 +100,30 @@ class ElasticUtils {
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
+3
-2
@@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery;
|
||||
import com.fangjia.dal.SortParam;
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
import com.fangjia.dal.mongo.model.DistrictFacilitys;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.service.bo.DistrictInfo;
|
||||
@@ -540,10 +541,10 @@ public interface DistrictService extends ModeService<DistrictKey, DistrictBo> {
|
||||
*
|
||||
* @param city
|
||||
* @param region
|
||||
* @param masterDistrictNname
|
||||
* @param masterDistrictName
|
||||
* @return
|
||||
*/
|
||||
List<District> findSlaves(String city, String region, String masterDistrictNname);
|
||||
List<District> findSlaves(String city, String region, String masterDistrictName);
|
||||
|
||||
/**
|
||||
* 分期操作时设置此小区为人工操作标识
|
||||
|
||||
+2
-3
@@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<District> findSlaves(String city, String region, String masterDistrictNname) {
|
||||
public List<District> findSlaves(String city, String region, String masterDistrictName) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("visible", 0));
|
||||
params.add(new QueryParam("city", city));
|
||||
params.add(new QueryParam("region", region));
|
||||
params.add(new QueryParam("masterDistrictNname", masterDistrictNname));
|
||||
params.add(new QueryParam("masterDistrictName", masterDistrictName));
|
||||
return districtDao.findAllByParams(params);
|
||||
}
|
||||
|
||||
@@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService {
|
||||
params.add(new QueryParam("$sr", new SrQuery(query)));
|
||||
return districtDao.findAllByParams(params);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/6/11
|
||||
*/
|
||||
public interface AliasFangService {
|
||||
|
||||
/**
|
||||
* 查询搜房别名与名称关系
|
||||
*
|
||||
* @param city
|
||||
* @param region
|
||||
* @param names
|
||||
* @return
|
||||
*/
|
||||
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.dao.AliasFangDao;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/6/11
|
||||
*/
|
||||
public class AliasFangServiceImpl implements AliasFangService{
|
||||
|
||||
private AliasFangDao aliasFangDao;
|
||||
|
||||
@Override
|
||||
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
|
||||
return aliasFangDao.findFangByNameAndAlias(city, region, names);
|
||||
}
|
||||
|
||||
public void setAliasFangDao(AliasFangDao aliasFangDao) {
|
||||
this.aliasFangDao = aliasFangDao;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user