Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_bohai
# Conflicts: # projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java
This commit is contained in:
@@ -13,7 +13,7 @@ import java.util.*;
|
||||
*/
|
||||
public class BaiDuApiUtils {
|
||||
|
||||
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
|
||||
private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX";
|
||||
public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
|
||||
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
|
||||
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
|
||||
|
||||
@@ -25,6 +25,7 @@ import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import com.hankcs.hanlp.HanLP;
|
||||
import com.hankcs.hanlp.dictionary.py.Pinyin;
|
||||
import com.sun.javafx.scene.transform.TransformUtils;
|
||||
import net.sf.json.JSON;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONNull;
|
||||
@@ -1435,6 +1436,25 @@ public class DataUtil {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
public static List<String> matchAll(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null;
|
||||
}
|
||||
List list = new ArrayList();
|
||||
Matcher matcher = pattern.matcher(str);
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static boolean isNumeric(String str) {
|
||||
try {
|
||||
Double.parseDouble(str);
|
||||
@@ -1805,11 +1825,26 @@ public class DataUtil {
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(getFirstChar("文裕花园"));
|
||||
System.out.println(getFirstChar("文怡花园"));
|
||||
public static boolean isChineseChar(String c) {
|
||||
if (StringUtils.isNotBlank(c)) {
|
||||
String temp = String.valueOf(c);
|
||||
Pattern pattern = Pattern.compile("[\\u4e00-\\u9fa5\\w]+");
|
||||
Matcher matcher = pattern.matcher(temp);
|
||||
return matcher.matches();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
System.out.println(FourUpSixInto("258.393", 2));
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(isChineseChar("6文怡a花园"));
|
||||
// System.out.println(getFirstChar("文怡花园"));
|
||||
//
|
||||
// System.out.println(FourUpSixInto("258.393", 2));
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
|
||||
|
||||
public static void main(String[] args) {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
System.out.println(random(0, 10));
|
||||
System.out.println(random(-9, 9));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+32
-1
@@ -1,5 +1,9 @@
|
||||
package com.fangjia.coresite.controllers;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
@@ -7,6 +11,7 @@ import java.util.List;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fangjia.coresite.model.JsonModel;
|
||||
@@ -37,7 +42,7 @@ public class SaleLeadsController {
|
||||
}
|
||||
|
||||
public Object doAddAction(SaleLeadsModel model, HttpServletRequest req,
|
||||
HttpServletResponse res) {
|
||||
HttpServletResponse res) throws UnsupportedEncodingException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
|
||||
|
||||
model.setViewType("redirect");
|
||||
String refUrl = model.getBackUrl();
|
||||
@@ -122,6 +127,8 @@ public class SaleLeadsController {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
saleLeads = converValue(saleLeads);
|
||||
saleLeadsService.save(saleLeads);
|
||||
|
||||
if(StringUtils.isNotBlank(saleLeads.getBusiness()) && saleLeads.getBusiness().contains("_")){
|
||||
@@ -162,6 +169,30 @@ public class SaleLeadsController {
|
||||
model.setViewParam("location", url);
|
||||
return model;
|
||||
}
|
||||
|
||||
private SaleLeads converValue(SaleLeads saleLeads) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, UnsupportedEncodingException {
|
||||
//
|
||||
Field[] fields = saleLeads.getClass().getDeclaredFields();
|
||||
for(Field s : fields){
|
||||
String name = s.getName();
|
||||
String type = s.getGenericType().toString();
|
||||
if(type.contains("java.lang.String")){
|
||||
String fildName = name.substring(0,1).toUpperCase()+name.substring(1);
|
||||
Method m = saleLeads.getClass().getMethod("get"+fildName);
|
||||
String value = (String)m.invoke(saleLeads);
|
||||
if(StringUtils.isNotBlank(value)){
|
||||
String s1 = value.replaceAll("\\pP|\\pS", "");
|
||||
if(!DataUtil.isChineseChar(s1)){
|
||||
String newValue = new String(value.getBytes("ISO8859-1"), "UTF-8");
|
||||
m = saleLeads.getClass().getMethod("set"+fildName,String.class);
|
||||
m.invoke(saleLeads, newValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return saleLeads;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public Object addDownInfoAction(SaleLeadsModel model, HttpServletRequest req,
|
||||
|
||||
@@ -102,10 +102,10 @@ public class Loudong implements MongoEntity {
|
||||
*/
|
||||
private String nameInSeaweed;
|
||||
/**
|
||||
* 小区编号
|
||||
* seaweed中的小区id
|
||||
*/
|
||||
private String idInSeaweed;
|
||||
|
||||
|
||||
@Column(name = "remark")
|
||||
public String getRemark() {
|
||||
return remark;
|
||||
|
||||
@@ -38,6 +38,7 @@ public class OpenConfig extends Configuration {
|
||||
|
||||
private final String ipdatfile;
|
||||
private final String baiduAk;
|
||||
private final String defaultCity;
|
||||
|
||||
private final boolean isOpenAllAuthority;
|
||||
|
||||
@@ -54,6 +55,7 @@ public class OpenConfig extends Configuration {
|
||||
baiduAk = configuration.get("fangjia.baidu.ak");
|
||||
isOpenAllAuthority = configuration.getBoolean("fangjia.isOpenAllAuthority", false);
|
||||
enabledAddressMatch = configuration.getBoolean("fangjia.addressMatch.enabled",false);
|
||||
defaultCity = configuration.get("fangjia.default.city");
|
||||
}
|
||||
|
||||
public static OpenConfig getConfig() {
|
||||
@@ -134,4 +136,7 @@ public class OpenConfig extends Configuration {
|
||||
return enabledAddressMatch;
|
||||
}
|
||||
|
||||
public String getDefaultCity() {
|
||||
return defaultCity;
|
||||
}
|
||||
}
|
||||
|
||||
+226
@@ -89,6 +89,7 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService;
|
||||
import com.fangjia.service.open.WaitCrawlKeywordsService;
|
||||
import com.fangjia.service.sql.FjtEvaluationService;
|
||||
import com.fangjia.service.web.AddressService;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
@Authority(role = Role.USER)
|
||||
public class AddressController {
|
||||
@@ -106,6 +107,7 @@ public class AddressController {
|
||||
private HouseLoudongService houseLoudongService;
|
||||
private FjtEvaluationService fjtEvaluationService;
|
||||
private AddressMatchProxyService addressMatchProxyService;
|
||||
|
||||
private final static String AK = OpenConfig.getConfig().getBaiduAk();
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(AddressController.class);
|
||||
|
||||
@@ -117,6 +119,8 @@ public class AddressController {
|
||||
private static final String GRADE_NATION = "全国评分";
|
||||
private static final String GRADE_CONSUME = "消费评分";
|
||||
|
||||
public final static OpenConfig CONFIG = OpenConfig.getConfig();
|
||||
|
||||
/**
|
||||
* @deprecated see parse2Action
|
||||
* @param model
|
||||
@@ -1705,6 +1709,228 @@ public class AddressController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 地址标准化接口V2
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object standard2Action(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 五级地址标准化接口
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object fiveDivisionStructureAction(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 五级行政区划V2地址标准化接口
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object fiveDivisionStructure2Action(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 详细地址标准化接口
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object divisionStructureAction(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 地址标准化专用测试接口
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object structureAndCompleteAction(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object similarityAction(AddesssMatchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
|
||||
|
||||
+6
-12
@@ -665,18 +665,6 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
String userId = FangjiaApiUtil.parseUserId(model.getToken());
|
||||
|
||||
/**超级用户和上海交大账户不控制权限*/
|
||||
if (!"1007497".equals(userId) && !"1007632".equals(userId) && !"1007548".equals(userId)
|
||||
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
|
||||
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
|
||||
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
|
||||
&& !"1007747".equals(userId) && !"1007757".equals(userId)
|
||||
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel());
|
||||
return mav;
|
||||
}
|
||||
|
||||
District district = null;
|
||||
if (StringUtils.isNotBlank(model.getId())){
|
||||
if(ObjectId.isValid(model.getId())){
|
||||
@@ -961,6 +949,12 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
|
||||
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
|
||||
if(loudong != null && loudong.getFloors() >= model.getFloor()){
|
||||
model.setMaxFloor(loudong.getFloors());
|
||||
}
|
||||
}
|
||||
model.setRegion(district.getRegion());
|
||||
model.setDistrictName(district.getName());
|
||||
if (StringUtils.isBlank(model.getPropertyType())) {
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.open.interceptors.Authority;
|
||||
import com.fangjia.open.interceptors.Role;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class requestController {
|
||||
|
||||
/**
|
||||
* »ñÈ¡ÇëÇóµÄrequestºÍIPÐÅÏ¢
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
public Object requestAction(HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
String ip = AnalyticsUtil.getIpAddress(request);
|
||||
Map<String, String> map = new HashMap<String, String>();
|
||||
Enumeration headerNames = request.getHeaderNames();
|
||||
while (headerNames.hasMoreElements()) {
|
||||
String key = (String) headerNames.nextElement();
|
||||
String value = request.getHeader(key);
|
||||
map.put(key, value);
|
||||
}
|
||||
map.put("ip", ip);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(map));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
@@ -100,4 +100,5 @@ fangjia.jms.email.queue=email.queue
|
||||
|
||||
fangjia.redis_nodes=192.168.0.79:6379,192.168.0.79:6380,192.168.0.79:6381,192.168.0.81:6382,192.168.0.81:6383,192.168.0.81:6384
|
||||
|
||||
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
|
||||
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
|
||||
fangjia.default.city = \u91CD\u5E86\u5E02
|
||||
@@ -1,8 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlrewrite>
|
||||
<global-params>
|
||||
<param value="st" name="sort" type="attach"/>
|
||||
<param value="pn" name="pager.currentPage" type="attach"/>
|
||||
<!-- <param value="st" name="sort" type="attach"/>-->
|
||||
<!-- <param value="pn" name="pager.currentPage" type="attach"/>-->
|
||||
</global-params>
|
||||
<rules>
|
||||
<rule controller="city" action="condition2" channel="v2">
|
||||
@@ -33,5 +33,15 @@
|
||||
<ladder>/property/avgPrice</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
<rule controller="address" action="standard2" channel="v2">
|
||||
<ladders>
|
||||
<ladder>/address/standard</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
<rule controller="address" action="fiveDivisionStructure2" channel="v2">
|
||||
<ladders>
|
||||
<ladder>/address/fiveDivisionStructure</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
</rules>
|
||||
</urlrewrite>
|
||||
@@ -494,5 +494,32 @@
|
||||
<artifactId>msgpack</artifactId>
|
||||
<version>0.6.12</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.asprise.ocr</groupId>
|
||||
<artifactId>java-ocr-api</artifactId>
|
||||
<version>[15,)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.tess4j</groupId>
|
||||
<artifactId>tess4j</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+49
-35
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", 1)
|
||||
// q.put("name","ÁÉÔ´Ò»´å")
|
||||
def dbcursor = LOU_DONG_STAT.find(q)
|
||||
def map = [:]
|
||||
def set = new HashSet()
|
||||
for(def db : dbcursor){
|
||||
|
||||
def list = map.get(db.region+"##"+db.name)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(db.region+"##"+db.name, list)
|
||||
}
|
||||
list << db
|
||||
if(db.house_num && db.house_num!="--"){
|
||||
def key = db.region+"##"+db.name +"##"+db.house_num
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
list << db
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
|
||||
def rets = []
|
||||
def keys = []
|
||||
distMap.each{k,dist->
|
||||
def buildings = buildingMap.get(k)
|
||||
if(buildings){
|
||||
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(obj.region)
|
||||
if(areaData){
|
||||
ret << cityCodeDatas.get(obj.city)
|
||||
ret << obj._id.toString()
|
||||
ret << obj.house_num
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << areaData.district_code
|
||||
ret << dist._id.toString()
|
||||
if(obj.build_year){
|
||||
ret << obj.build_year
|
||||
}else{
|
||||
def key = dist._id.toString() +"##"+obj.house_num
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
ret << cityCodeDatas.get(obj.city)
|
||||
ret << obj._id.toString()
|
||||
ret << obj.house_num
|
||||
ret << obj.house_num
|
||||
ret << ""
|
||||
}
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
ret << areaData.district_code
|
||||
ret << dist._id.toString()
|
||||
if(obj.build_year){
|
||||
ret << obj.build_year
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
// if(obj.elevators && obj.elevators>=1){
|
||||
// ret << "1"
|
||||
// }else{
|
||||
// ret << "0"
|
||||
// }
|
||||
ret << ""
|
||||
ret << ""
|
||||
// if(obj.elevators && obj.elevators>=1) {
|
||||
// ret << obj.elevators//µçÌÝÊý
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << "1"
|
||||
rets << ret
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << "1"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+5
-4
@@ -54,10 +54,10 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
|
||||
for(def city : citys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
println citys.unique().size()
|
||||
// for(def city : citys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// println citys.unique().size()
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("status", 0)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+27
-4
@@ -149,8 +149,8 @@ class CcbCommonUtils {
|
||||
|
||||
public static def findTradesDetailById(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
// q.put("city", dist.city)
|
||||
// q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
@@ -161,7 +161,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -367,7 +367,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","长隆小区")
|
||||
// q.put("name","ÁÉÔ´Ò»´å")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
|
||||
|
||||
@@ -488,4 +488,27 @@ class CcbCommonUtils {
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
public static def findDistrictSeqno(){
|
||||
def sql = "select cityid,commid,seqno from bigdata_position_seqno";
|
||||
def map = new HashMap()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def detailMap = map.get(r.cityid)
|
||||
if(!detailMap){
|
||||
detailMap = [:]
|
||||
map.put(r.cityid, detailMap)
|
||||
}
|
||||
detailMap.put(r.commid, r.seqno)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
public static def findMaxSeqno(def cityid){
|
||||
def sql = "select max(seqno) as max_seqno from bigdata_position_seqno where cityid='${cityid}'"
|
||||
def maxSeqno = null;
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
maxSeqno = r.max_seqno
|
||||
});
|
||||
return maxSeqno
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jiangnanblank.SFTP
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class CcbFtpTestUtils {
|
||||
|
||||
static def ftpServer = "192.168.0.86"
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "ccb"
|
||||
static def ftpPassword = "ccb!@#"
|
||||
static def ftpPath = "/home/ccb/output/"
|
||||
|
||||
static def ftpPath_rh = "/home/ccb_rh/output/"
|
||||
|
||||
static SFTP sftp = new SFTP();
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
|
||||
|
||||
static def ftpUpload(type,outputPath_dist, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
if(type && type=="rh"){
|
||||
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
|
||||
}else{
|
||||
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static def delete(path){
|
||||
if(path.startsWith("/")){
|
||||
sftp.deleteSFTP(path, sftpChannel)
|
||||
}else{
|
||||
sftp.deleteFile(path)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def reConnect(){
|
||||
if(sftpChannel == null){
|
||||
println "sftpChannel 为空,需重连....."
|
||||
if(sftp == null){
|
||||
sftp = new SFTP();
|
||||
}
|
||||
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
}else{
|
||||
println "sftpChannel 正常....."
|
||||
}
|
||||
return sftpChannel;
|
||||
}
|
||||
|
||||
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
|
||||
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
|
||||
|
||||
def data = new FileInputStream(new File(dataFileName))
|
||||
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+9
-9
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()
|
||||
task.delDealByPe()
|
||||
task.deleteThirdPartyData()
|
||||
// task.delDealByPe()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -370,26 +370,26 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = ["东莞"] //,"杭州""广州",
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202110
|
||||
def month = 202203
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202111)
|
||||
def rets = findPeGteThirdParty(city, 202204)
|
||||
def total = 0
|
||||
for(def m : rets){
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
println city +" "+total
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>50"
|
||||
def rets = []
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.commid = r.commid
|
||||
map.price = r.target_avg_price
|
||||
|
||||
+178
-2
@@ -1,15 +1,20 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -37,15 +42,74 @@ class DistrictDelete {
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
// task.gzRun();
|
||||
// task.deleteDistrict()
|
||||
task.deleteCommPrice()
|
||||
// task.deleteCommPrice()
|
||||
// task.distHouseCountStat()
|
||||
task.addThirdPartyStat()
|
||||
// task.processRegion()
|
||||
}
|
||||
def processRegion(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
|
||||
if(d['B']){
|
||||
def region = RegionData.formatRegion("重庆",d['B'])
|
||||
if(region){
|
||||
d['B'] = region
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addThirdPartyStat(){
|
||||
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
|
||||
def idMaps = [:]
|
||||
for(def d : datas){
|
||||
def id = d['A']
|
||||
idMaps.put(id, d['D'])
|
||||
}
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
def totalCount = new AtomicLong()
|
||||
def map = findThirdPartyPrice("上海")
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def thirdLists = Collections.synchronizedList(new ArrayList());
|
||||
thirdLists << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = idMaps.get(d._id.toString())
|
||||
if (!count) {
|
||||
def dist = CommonDatautils.findDistrictById(d._id.toString())
|
||||
def priceMap = map.get(dist.region + "##" + dist.name)
|
||||
|
||||
if (priceMap) {
|
||||
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
|
||||
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
|
||||
}
|
||||
}
|
||||
// println totalCount.get() + " " + d.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -60,6 +124,40 @@ class DistrictDelete {
|
||||
}
|
||||
}
|
||||
|
||||
def distHouseCountStat(){
|
||||
def city = "重庆"
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def distCount = districts.size()
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
rets << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
|
||||
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
|
||||
if(count){
|
||||
rets << [dist._id.toString(), dist.region, dist.name, count]
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
|
||||
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
@@ -259,4 +357,82 @@ class DistrictDelete {
|
||||
sqlInstance.execute(sql)
|
||||
println "ɾ³ý£º"+tableName+" "+commid
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def prices = map.get(key)
|
||||
if(!prices){
|
||||
prices = []
|
||||
map.put(key, prices)
|
||||
}
|
||||
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
prices << db.zhugezhaofang_esf_price
|
||||
}
|
||||
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
prices << db.anjuke_esf_price
|
||||
}
|
||||
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
prices << db.get("58tc_esf_price")
|
||||
}
|
||||
if(db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
prices << db.soufang_esf_price
|
||||
}
|
||||
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
prices << db.lianjia_esf_price
|
||||
}
|
||||
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
prices << db.cityhouse_esf_price
|
||||
}
|
||||
|
||||
if(db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
prices << db.maitian_esf_price
|
||||
}
|
||||
|
||||
if(db.beike_esf_price && db.beike_esf_price>3000){
|
||||
prices << db.beike_esf_price
|
||||
}
|
||||
|
||||
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
prices << db.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
prices << db.zhongyuan_esf_price
|
||||
}
|
||||
|
||||
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
prices << db.tuboshi_esf_price
|
||||
}
|
||||
|
||||
if( db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
prices << db.qfang_esf_price
|
||||
}
|
||||
|
||||
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
prices << db.fangtan_esf_price
|
||||
}
|
||||
|
||||
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
prices << db.fangduoduo_esf_price
|
||||
}
|
||||
|
||||
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
prices << db.get("5i5j_esf_price")
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-03-1", "2022-04-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+20
-21
@@ -21,26 +21,25 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
def run() {
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// //不会变
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportBigdataDistrictData()
|
||||
//
|
||||
// exportGuidedPrice()//随人工改变
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
//
|
||||
// //处理小区情况
|
||||
// exportBigdataCommunity()
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
//不会变
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()
|
||||
exportBigdataDistrictData()
|
||||
|
||||
// exportThirdPartyPrice2(lastMonth)
|
||||
// exportBigdataForsale("2021-12-19")
|
||||
// exportBigdataDeal("2021-12-01")
|
||||
exportGuidedPrice()//随人工改变
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()
|
||||
exportCommunityStaging()
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-03-19")
|
||||
exportBigdataDeal("2022-03-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
@@ -56,11 +55,11 @@ class ExportOracleData {
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// def curDate = curMonth.toString()+"28"
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//ok文件
|
||||
exportLastFile()
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
|
||||
class ExportOracleEvaluationData {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportOracleEvaluationData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
//计算结果相关
|
||||
exportAvmEvaluationGJNB(curMonth)
|
||||
exportAVMEvaluationItemGJNB(curMonth)
|
||||
|
||||
exportCityPrice(curMonth)
|
||||
exportRegionPrice(curMonth)
|
||||
|
||||
exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
exportCommunityBasePrice(curMonth)
|
||||
|
||||
exportCommunityPdescribeGj(curMonth)
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// ok文件
|
||||
exportOkFile()
|
||||
}
|
||||
|
||||
def exportAvmEvaluationGJNB(yearMonth){
|
||||
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
|
||||
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
|
||||
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.COMMUNITY_NUM
|
||||
data << r.HAS_PRICE_COMMUNITY_NUM
|
||||
data << r.PRICE_COVERAGE
|
||||
data << r.ARTIFICIAL_CALC_NUM
|
||||
data << r.ARTIFICIAL_CALC_COVERAGE
|
||||
data << r.SAMPLE_CALC_NUM
|
||||
data << r.SAMPLE_CALC_COVERAGE
|
||||
data << r.ANN_CALC_NUM
|
||||
data << r.ANN_CALC_COVERAGE
|
||||
data << r.TEST_NUM
|
||||
data << r.TEST_COMM_NUM
|
||||
data << r.TEST_COMM_COVERAGE
|
||||
data << r.PPE15_COVERAGE
|
||||
data << r.RATE_MEDIAN
|
||||
data << r.COEFFICIENT
|
||||
data << r.DEAL_TEST_CASE_NUM
|
||||
data << r.DEAL_TEST_COMM_NUM
|
||||
data << r.DEAL_TEST_COMM_COVERAGE
|
||||
data << r.DEAL_PPE15_COVERAGE
|
||||
data << r.DEAL_COEFFICIENT
|
||||
data << r.DEAL_RATE_MEDIAN
|
||||
data << r.THIRD_TEST_CASE_NUM
|
||||
data << r.THIRD_TEST_COMM_NUM
|
||||
data << r.THIRD_TEST_COMM_COVERAGE
|
||||
data << r.THIRD_PPE15_COVERAGE
|
||||
data << r.THIRD_COEFFICIENT
|
||||
data << r.THIRD_RATE_MEDIAN
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
|
||||
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
|
||||
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportAVMEvaluationItemGJNB(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
|
||||
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.CONFIDENCE
|
||||
data << r.TARGET_AVG_PRICE
|
||||
data << r.DEALDATE
|
||||
data << r.UNIT_PRICE
|
||||
data << r.BUILDAREA
|
||||
data << r.FITMENT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.DATA_TYPE
|
||||
data << r.RATE
|
||||
data << r.PE
|
||||
data << r.CREATETIME
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePrice(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePriceGj(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCityPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "CITY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "CITY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportRegionPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//
|
||||
// for(def city : citys) {
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
|
||||
// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.CITYID
|
||||
data << r.CITY_NAME
|
||||
data << r.DISTRICT_CODE
|
||||
data << r.DISTRICT_NAME
|
||||
data << r.COMMUNITY_NAME
|
||||
data << r.PQ_NAME
|
||||
data << r.BASE_PRICE
|
||||
data << r.CONFIDENCE_LEVEL
|
||||
data << r.CNT
|
||||
data << r.MIN_PRICE
|
||||
data << r.MEDIANPRICE
|
||||
data << r.MEANPRICE
|
||||
data << r.MAX_PRICE
|
||||
data << r.TIME
|
||||
data << r.FIRSTDATAT
|
||||
data << r.THIRDDATA
|
||||
data << r.RELATIVEERROR
|
||||
data << r.SKEWNESS
|
||||
data << r.PQ_PRICE_RANGE
|
||||
data << r.DISTRICT_PRICE_RANGE
|
||||
data << r.DESCRIPTION
|
||||
data << r.PRIV_BASE_PRICE
|
||||
data << r.IS_ARTTFICIAL
|
||||
data << r.PRIV_FINAL_PRICE
|
||||
data << r.CREATE_DATE
|
||||
data << r.PRICE
|
||||
data << r.PRICE_SOURCE
|
||||
data << r.CASE_MD_PRICE
|
||||
data << r.CASE_AVG_PRICE
|
||||
data << r.INDEX_LABEL
|
||||
data << r.MAIN_PROPERTY
|
||||
data << r.COMMTYPE
|
||||
data << r.NEW_DEAL_PRICE
|
||||
data << r.NEW_DEAL_DATE
|
||||
data << r.PRICE_MOM_RATIO
|
||||
data << r.MAX_MOM_RATIO
|
||||
data << r.PRICE_YOY_RATIO
|
||||
data << r.SATUR_RATIO
|
||||
data << r.NEIGHBOR_PRICE
|
||||
allRows << data
|
||||
});
|
||||
println allRows.size()
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
// }
|
||||
println "community_p_describe_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "community_p_describe_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommCaseDetailGj(month){
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.NETLISTING_ID
|
||||
data << r.PRICE_DATE
|
||||
data << r.CREATE_DATE
|
||||
allRows << data
|
||||
if(allRows.size()%200000==0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
allRows.clear()
|
||||
}
|
||||
});
|
||||
if(allRows.size()>0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
println "comm_case_detail_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "comm_case_detail_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOkFile(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def dataCodeFileName = "/ccb/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+7
-6
@@ -23,9 +23,10 @@ class LoadGuide {
|
||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
|
||||
|
||||
}
|
||||
def match(def filename){
|
||||
@@ -48,15 +49,15 @@ class LoadGuide {
|
||||
def datas = ExcelUtils.read(filePath,0, 1)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
if(d['D']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.city = d['D']
|
||||
o.region = d['A']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.match_city = d['D']
|
||||
o.match_region = d['E']
|
||||
o.match_name = d['F']
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
|
||||
+92
-23
@@ -224,18 +224,43 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
}
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
|
||||
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['C']){
|
||||
def oldcount = dealMap.get(d['C'])
|
||||
d['T'] = oldcount
|
||||
def casecount = testMap.get(d['C'])
|
||||
d['U'] = casecount
|
||||
println d
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202203)
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// def oldcount = dealMap.get(d['C'])
|
||||
// d['X'] = oldcount
|
||||
// def casecount = testMap.get(d['C'])
|
||||
// d['Y'] = casecount
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
def rows = findTradePpeData(sqlInstance236)
|
||||
// rows.unique()
|
||||
def map=findTradeDetail(sqlInstance236,null)
|
||||
def datas=[]
|
||||
map.each{key,row->
|
||||
if(!rows.contains(key)){
|
||||
datas<<row
|
||||
}
|
||||
},0)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def datas=[]
|
||||
//cityCodeDataMap.each{city,code->
|
||||
// def rows = findTradePpeData(sqlInstance236, 202203,code)
|
||||
// def alltrades=findTradeDetail(sqlInstance236,code)
|
||||
// alltrades.each{it->
|
||||
// def commid=it[0]
|
||||
// if(!rows.contains(commid)){
|
||||
// datas<<it
|
||||
// }
|
||||
// }
|
||||
// println datas.size()
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
||||
// if(d['I'] && d['K']){
|
||||
@@ -271,19 +296,24 @@ ExcelUtils.modify(fileName,{d->
|
||||
//},0)
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
|
||||
// def flag = d['M']
|
||||
// if(flag==0.0){
|
||||
// if(d['D']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['A']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['D']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def region = d['B']
|
||||
@@ -553,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findTradePpeData(def sqlInstance){
|
||||
def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'";
|
||||
def rows = new HashSet()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
rows<<r.commid+"##"+r.buildarea+"##"+r.target_avg_price+"##"+r.dealdate
|
||||
});
|
||||
return rows
|
||||
}
|
||||
def findTradeDetail(def sqlInstance, def cityid){
|
||||
def sql = "select commid,cityname,distname,commname,buildarea,dealunitprice,to_char(dealdate,'yyyy-mm-dd') as dealdate from bigdata_deal_gj where dealdate>=to_date('2022-02-01','yyyy-MM-dd')";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate
|
||||
def row=[]
|
||||
row<<r.commid
|
||||
row<<r.cityname
|
||||
row<<r.distname
|
||||
row<<r.commname
|
||||
row<<r.buildarea
|
||||
row<<r.dealunitprice
|
||||
map.put(key,row)
|
||||
});
|
||||
return map
|
||||
}
|
||||
def findEstateType(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
|
||||
def count = 0
|
||||
@@ -777,6 +829,23 @@ def getStreetInfo(def city, def glocation){
|
||||
//}
|
||||
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
|
||||
|
||||
//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
|
||||
//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
|
||||
//
|
||||
//def keys = []
|
||||
//for(def d : datas2){
|
||||
// keys << d['A']+"##"+d['C']+"##"+d['B']
|
||||
//}
|
||||
//
|
||||
//def index = 0
|
||||
//for(def d : datas1){
|
||||
// def k = d['A']+"##"+d['C']+"##"+d['B']
|
||||
// if(!keys.contains(k)){
|
||||
// index ++
|
||||
// println index +" "+k
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202110
|
||||
def endDate = 202112
|
||||
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202201
|
||||
def endDate = 202203
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+31
-9
@@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
@@ -48,6 +49,7 @@ class TradeStatLog {
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
// task.statTableCount()
|
||||
// task.filterTrade()
|
||||
|
||||
|
||||
}
|
||||
@@ -73,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-10-01"
|
||||
def date = "2021-11-01"
|
||||
def endDate = "2021-12-01"
|
||||
def newDate = "2022-01-01"
|
||||
def startDate = "2022-01-01"
|
||||
def date = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
def newDate = "2022-04-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -129,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -479,8 +481,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2021-12-01"
|
||||
def endDate = "2022-01-01"
|
||||
def startDate = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
@@ -561,7 +563,7 @@ class TradeStatLog {
|
||||
}
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
@@ -706,4 +708,24 @@ class TradeStatLog {
|
||||
|
||||
}
|
||||
|
||||
def filterTrade(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d->
|
||||
if(!d['J']){
|
||||
def count = findTradeById(d['A'])
|
||||
if(count){
|
||||
d['J'] = "其它原因过滤"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findTradeById(id){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(id))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+294
-2
@@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class CcbDistrictToDB {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
// task.run1()
|
||||
// task.meituanRun()
|
||||
// task.meituanSqrentRun()
|
||||
// task.updateDistrict()
|
||||
task.updateDongguangRegion()
|
||||
}
|
||||
|
||||
def updateDongguangRegion(){
|
||||
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
|
||||
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
|
||||
for(def d : dist){
|
||||
def t = d.c_date
|
||||
if(t.getTime()>cDate.getTime()){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateDistrict(){
|
||||
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
|
||||
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
|
||||
for(def d : dist){
|
||||
def t = d.c_date
|
||||
if(t.getTime()>cDate.getTime()){
|
||||
if(d.name.contains("(")){
|
||||
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
|
||||
def newName = d.name.replace(delstr,"")
|
||||
index++
|
||||
println index+" "+d.city+" "+d.name+" "+newName
|
||||
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
|
||||
if(!obj){
|
||||
println "update"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name", newName)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
|
||||
d.name = newName
|
||||
addPrice(d)
|
||||
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
|
||||
}else{
|
||||
println "delete"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def meituanSqrentRun(){
|
||||
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
|
||||
def city = d['F']
|
||||
def region = d['G']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion) {
|
||||
def name = d['H']
|
||||
name = nameFileter(name)
|
||||
if(name){
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def meituanRun(){
|
||||
def index = 0//美团需入库小区合并后.xlsx
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
|
||||
def city = d['F']
|
||||
def region = d['G']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['H']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
if(d['E']){
|
||||
formatAddress = AddressUtil.formatAddress(city, d['E'])
|
||||
}
|
||||
|
||||
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
|
||||
def map = findStandardObject2(city,region, name, d['E'], rets);
|
||||
if(map && map.address){
|
||||
formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
if(!formatAddress){
|
||||
if(map.address.contains("交叉口")){
|
||||
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
|
||||
}else if(map.address.contains("交叉")){
|
||||
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
|
||||
|
||||
println index+" "+data
|
||||
|
||||
def stop = false
|
||||
while (!stop){
|
||||
if(data._id.toString()){
|
||||
addPrice(data)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
},0);
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def run1(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
@@ -71,6 +289,7 @@ class CcbDistrictToDB {
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
@@ -221,4 +440,77 @@ class CcbDistrictToDB {
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+82
-1
@@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
class DistrictMarge {
|
||||
static main(args) {
|
||||
def task = new DistrictMarge()
|
||||
task.run()
|
||||
//task.run()
|
||||
task.repeatAddress()
|
||||
}
|
||||
|
||||
def repeatAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
|
||||
findRepead(c, districts)
|
||||
}
|
||||
}
|
||||
def findRepead(city, districts){
|
||||
def map = [:]
|
||||
for(def d : districts){
|
||||
if(d.name_cat == 0 && d.lng){
|
||||
if(d.address && d.address.contains("号")){
|
||||
def list = map.get(d.address)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(d.address, list)
|
||||
}
|
||||
list << d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def newMap = [:]
|
||||
map.each {k,vals->
|
||||
if(vals.size()>=2){
|
||||
vals.sort{o1,o2->
|
||||
return o1.name.size() <=> o2.name.size()
|
||||
}
|
||||
newMap.put(k,vals)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
newMap.each {k,vals->
|
||||
def names = []
|
||||
for(def d : vals){
|
||||
def newKey = d.name
|
||||
if(d.name.contains("小区")){
|
||||
newKey = d.name.toString().replace("小区", "")
|
||||
}
|
||||
if(!names.contains(newKey)){
|
||||
names << newKey
|
||||
}else{
|
||||
d.flag = "删除"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId( d._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def values = newMap.values()
|
||||
def allRets = []
|
||||
for(def list : values){
|
||||
allRets.addAll(list)
|
||||
}
|
||||
def results = []
|
||||
def delcount = 0
|
||||
for(def r : allRets){
|
||||
def rows = []
|
||||
rows << r._id.toString()
|
||||
rows << r.city
|
||||
rows << r.name
|
||||
rows << r.address
|
||||
rows << r.name_cat
|
||||
if(r.flag){
|
||||
rows << r.flag
|
||||
delcount++
|
||||
}
|
||||
results << rows
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
|
||||
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def rets = getDatas()
|
||||
def deleteIndex = 0
|
||||
|
||||
+163
-20
@@ -1,24 +1,123 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", ZS_MONGODB_URL)
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
|
||||
if(d['B']){
|
||||
//def avg_price =CommonDatautils.queryCityPrice(d['B'])
|
||||
//d['C'] = avg_price
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("area_name",new BasicDBObject( '$regex' , d['B'].replace("州","")+/.*$/))
|
||||
// def obj = COMMUNITY_CONN.findOne(q)
|
||||
// if(obj){
|
||||
// d['D'] = obj.area_code
|
||||
// println obj.area_name +" "+obj.area_code
|
||||
// }
|
||||
|
||||
//def obj = findDistrictPriceCount(d['B'], DateUtils.parseFromTightByPattern("2022-03-13","yyyy-MM-dd"), CITY_INFO)
|
||||
// d['D'] = CommonDatautils.findAllDistrictCount("district", d['B'])
|
||||
// d['E'] = findHouseCount(d['B'], 20210101, 20211231)
|
||||
//
|
||||
// def startDate = DateUtils.parseFromTightByPattern("2021-01-01", "yyyy-MM-dd")
|
||||
// def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
// def deals = findTradesDetailById(d['B'], startDate,endDate, DEAL_PRICE_COLL)
|
||||
// d['F'] = deals.size()
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
d['G'] = findCcbDistCount(cityCodeDataMap.get(d['B']), sqlInstance)
|
||||
}
|
||||
},0)
|
||||
|
||||
def findCcbDistCount(cityid, sqlInstance){
|
||||
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityId='${cityid}' and status=0 ";
|
||||
def total = null
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
total = r.count
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetailById(city, startDate, endDate, DEAL_PRICE_COLL){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
def findDistrictPriceCount(def city, def weekend, def CITY_INFO){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def obj = CITY_INFO.findOne(city)
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']) {
|
||||
@@ -433,23 +532,67 @@ def city = "
|
||||
// }
|
||||
//})
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
if(dist){
|
||||
def alias = dist.mapping_alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(alias && !alias.contains(d['C'])){
|
||||
index++
|
||||
println index
|
||||
alias << d['C']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mapping_alias",alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
// if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(alias && !alias.contains(d['C'])){
|
||||
// index++
|
||||
// println index
|
||||
// alias << d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
|
||||
// if(d['A']){
|
||||
// try{
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
|
||||
// if(dist){
|
||||
// d['B'] = dist.city
|
||||
// d['C'] = dist.region
|
||||
// d['D'] = dist.name
|
||||
//
|
||||
// def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
|
||||
// d['E'] = price
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
// println d['A']
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def mappingAlias = dist.mapping_alias
|
||||
// if(mappingAlias){
|
||||
// if(mappingAlias.contains(d['B'])){
|
||||
// mappingAlias.remove(d['B'])
|
||||
// DBObject udata = new BasicDBObject();
|
||||
// udata.put("mapping_alias", mappingAlias);
|
||||
// udata.put("m_date", new Date());
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d['A']+"无映射别名"
|
||||
// }
|
||||
// }else{
|
||||
// println d['A']+"映射别名为空"
|
||||
// }
|
||||
// }else{
|
||||
// println d['A']+"没找到"
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class MapAddressMatch {
|
||||
static main(args) {
|
||||
def task = new MapAddressMatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "北京"
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d ->
|
||||
index++
|
||||
println index +" "+ d['D']+" "+d['E']
|
||||
if(!d['G']){
|
||||
// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德")
|
||||
// if(gdData){
|
||||
// d['F'] = gdData.district
|
||||
// d['G'] = gdData.name
|
||||
// d['H'] = gdData.address
|
||||
// d['I'] = gdData.type
|
||||
// d['J'] = "高德"
|
||||
// }else{
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E'])
|
||||
def map = findStandardObject(city,d['D'],d['E'], rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']);
|
||||
map = findStandardObject(city,d['D'],d['E'], rets);
|
||||
}
|
||||
if(map){
|
||||
if(map.adname){
|
||||
d['F'] = map.adname
|
||||
}else if(map.region){
|
||||
d['F'] = map.region
|
||||
}
|
||||
d['G'] = map.name
|
||||
d['H'] = map.address
|
||||
if(map.type){
|
||||
d['I'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['I'] = map.estateType
|
||||
}
|
||||
|
||||
d['J'] = map.source
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
|
||||
continue;
|
||||
}
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
+49
-1
@@ -30,15 +30,63 @@ class MatchMapData {
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchMapData()
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.exportHouse()
|
||||
// task.addDistrictLevel()
|
||||
task.run4()
|
||||
|
||||
}
|
||||
|
||||
def run4() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
|
||||
def regionCode = null
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
regionCode = getRegionCode(newRegion)
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
d['F'] = "完全一致"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
} else {
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
d['F'] = "映射别名"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
} else {
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (rets) {
|
||||
if(rets.size()>1){
|
||||
d['F'] = "别名-多个"
|
||||
}else{
|
||||
d['F'] = "别名"
|
||||
}
|
||||
}
|
||||
if (rets) {
|
||||
dist = rets[0]
|
||||
}
|
||||
if (dist) {
|
||||
d['F'] = "完全一致"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def run3(){
|
||||
|
||||
def files = ["ºÏ·ÊÈÚºÏÆ¥Åä_1215.xlsx"]
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class TradeDistrictToDB {
|
||||
static main(args) {
|
||||
def task = new TradeDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def index = 0
|
||||
def keys = []
|
||||
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
|
||||
if(d['D']){
|
||||
def key = d['A']+"##"+d['D']+"##"+d['C']
|
||||
if(!keys.contains(key)){
|
||||
def formatAddress = null
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['F'])
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['G']){
|
||||
glat = d['H'].toString().toDouble() * 10000000000L
|
||||
glng = d['G'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def name = d['C'].replace("·","").replace("II","二")
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
keys << key
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], d['D'], name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",d['D'])
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['D'], d['C'], 4);
|
||||
println index+" "+data
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run() {
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if (newRegion) {
|
||||
def map = findMapData(city, newRegion, d['C'])
|
||||
if(map){
|
||||
d['D'] = newRegion
|
||||
d['E'] = map.name
|
||||
d['F'] = map.address
|
||||
d['G'] = map.lng
|
||||
d['H'] = map.lat
|
||||
d['I'] = map.lng2
|
||||
d['J'] = map.lag2
|
||||
}
|
||||
}
|
||||
}, 0);
|
||||
}
|
||||
|
||||
def findMapData(def city, def region, name){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
|
||||
def map = findStandardObject2(city, region, name, null, rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(city, region, name);
|
||||
map = findStandardObject2(city, region, name, null, rets);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+28
-47
@@ -55,6 +55,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
// addBigdataDistrictFJW()
|
||||
|
||||
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
||||
|
||||
def index = 0
|
||||
for (def city : citys) {
|
||||
index++
|
||||
@@ -101,7 +103,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
println index +" "+city +" 5.小区费用表"
|
||||
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 6.小区房屋用途表"
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
||||
println index +" "+ city + " 7.地理位置表"
|
||||
|
||||
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
||||
@@ -552,18 +554,17 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
// }
|
||||
}
|
||||
|
||||
def addBigdataPositionGJ(districts, def cityid){
|
||||
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
|
||||
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : districts) {
|
||||
index++
|
||||
def ret = []
|
||||
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
|
||||
ret << d.cityid
|
||||
ret << "5"
|
||||
ret << d.commid
|
||||
ret << "1"
|
||||
ret << index
|
||||
ret << seqno
|
||||
ret << d.lng
|
||||
ret << d.lat
|
||||
ret << ""
|
||||
@@ -589,49 +590,29 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
|
||||
println "BIGDATA_POSITION_FJW 数据文件输出完成......"
|
||||
}
|
||||
|
||||
// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
|
||||
//
|
||||
// def sqls = []
|
||||
// def index = 1
|
||||
// for(def d : districts){
|
||||
// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
|
||||
// index++
|
||||
// if(d.createtime){
|
||||
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
|
||||
// }else{
|
||||
// sql = sql + "null,"
|
||||
// }
|
||||
// if(d.updatetime){
|
||||
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
|
||||
// }else{
|
||||
// sql = sql + "null,'${d.sourcecode}')"
|
||||
// }
|
||||
// sqls << sql
|
||||
//
|
||||
// if (sqls.size() >= batchsize) {
|
||||
// try {
|
||||
// sqlInstance.withBatch(batchsize, { stmt ->
|
||||
// sqls.each { def s ->
|
||||
// stmt.addBatch s
|
||||
// }
|
||||
// })
|
||||
//
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// }
|
||||
// sqls.clear()
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// if (sqls.size() > 0) {
|
||||
// sqlInstance.withBatch(100, { stmt ->
|
||||
// sqls.each { def s ->
|
||||
// stmt.addBatch s
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
def findSeqno(cityid, commid, citySeqnoMap){
|
||||
def detailMap = citySeqnoMap.get(cityid)
|
||||
if(detailMap){
|
||||
def seqno = detailMap.get(commid)
|
||||
if(seqno){
|
||||
return seqno
|
||||
}else{
|
||||
seqno = CcbCommonUtils.findMaxSeqno(cityid)
|
||||
if(seqno){
|
||||
seqno = seqno + 1
|
||||
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
def insertSql = "INSERT INTO bigdata_position_seqno(CITYID,COMMID,SEQNO,CREATETIME) VALUES ('${cityid}','${commid}',${seqno},to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'))"
|
||||
sqlInstance.withBatch(batchsize, { stmt ->
|
||||
stmt.addBatch insertSql
|
||||
})
|
||||
return seqno
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println cityid+" ûÓиóÇÊеÄÐ¡ÇøÐòºÅ"
|
||||
}
|
||||
}
|
||||
|
||||
def addBigdataDistrictFJW(){
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
void process(Object args) {
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
//按天导(每天一个文件夹)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"))
|
||||
|
||||
if(curDate.toString().endsWith("10")){
|
||||
exportBigdataCommunityGJRH()
|
||||
@@ -471,7 +471,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(curDate.toString().endsWith("01")){
|
||||
if(curDate.toString().endsWith("28")){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
|
||||
+747
@@ -0,0 +1,747 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
runBatch(new ExportDistrictRongHeTest(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def fileName = "/ccb_rh/${curDate}"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}"
|
||||
// }
|
||||
CcbFtpTestUtils.delete(fileName)
|
||||
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
|
||||
|
||||
exportBigdataCommunityGJRH()
|
||||
exportBigdataBuildingFJW()
|
||||
exportBigdataCommfeeGj()
|
||||
|
||||
exportBigdataCommbuildstruGJ()
|
||||
exportBigdataCommhousetypeGJ()
|
||||
exportBigdataCommbuildyearGj()
|
||||
exportBigdataCommhouseuseGJ()
|
||||
exportBigdataPositionGJ()
|
||||
exportBigdataDistrictFJW()
|
||||
exportBigdataAliasFJW()
|
||||
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
exportEvaluateRate()
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportOk(curDate)
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
// def runByCity(){
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//
|
||||
//
|
||||
//// //检查ok文件是否上传
|
||||
//// exportAllBigdataDealFJW()
|
||||
//// for(def city : citys) {
|
||||
//// def cityCode = cityCodeDataMap.get(city)
|
||||
////
|
||||
// //按天导(一个文件夹下)
|
||||
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
// //按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
//
|
||||
//// exportBigdataCommunityGJRH(cityCode)
|
||||
//// exportBigdataBuildingFJW(cityCode)
|
||||
//// exportBigdataCommfeeGj(cityCode)
|
||||
////
|
||||
//// exportBigdataCommbuildstruGJ(cityCode)
|
||||
//// exportBigdataCommhousetypeGJ(cityCode)
|
||||
//// exportBigdataCommbuildyearGj(cityCode)
|
||||
//// exportBigdataCommhouseuseGJ(cityCode)
|
||||
//// exportBigdataPositionGJ(cityCode)
|
||||
//// exportBigdataDistrictFJW(cityCode)
|
||||
//// exportBigdataAliasFJW(cityCode)
|
||||
////
|
||||
//// exportBigdataBuildprojFjw(cityCode)
|
||||
//// exportBigdataCommlayoutpicFjw(cityCode)
|
||||
//// }
|
||||
//// exportEvaluateRate()
|
||||
//
|
||||
////
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
//
|
||||
// exportOk()
|
||||
// }
|
||||
|
||||
def exportBigdataCommunityGJRH(cityCode){
|
||||
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.COMMALIAS
|
||||
data << r.DISTID
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.ADDRESS
|
||||
data << r.LANDAREA
|
||||
data << r.BUILDAREA
|
||||
data << r.COMMTYPE
|
||||
data << r.IFNEW
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << ''
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
|
||||
}
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDSTRU
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommhousetypeGJ(cityCode){
|
||||
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.HOUSETYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommbuildyearGj(){
|
||||
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommfeeGj(){
|
||||
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.COMPID
|
||||
data << r.FEE
|
||||
data << r.HOUSEUSE
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.FEETYPE
|
||||
data << r.NOTE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommhouseuseGJ(){
|
||||
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSECNT
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataPositionGJ(){
|
||||
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.OBJECTTYPE
|
||||
data << r.OBJECTID
|
||||
data << r.POSITIONTYPE
|
||||
data << r.SEQNO
|
||||
data << r.LONGITUDE
|
||||
data << r.LATITUDE
|
||||
data << r.ALTITUDE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDistrictFJW(){
|
||||
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.DISTSPELL
|
||||
data << r.CITYID
|
||||
data << r.DISTCODE
|
||||
data << r.SEQ
|
||||
data << r.DISTTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataAliasFJW(){
|
||||
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.city_id
|
||||
data << r.ca_id
|
||||
data << r.address_id
|
||||
data << r.address_name
|
||||
data << r.status
|
||||
data << r.exp_dt
|
||||
data << r.version
|
||||
data << r.is_key_name
|
||||
data << r.data_source
|
||||
data << r.description
|
||||
data << r.alias_grade
|
||||
data << r.addr_type_cd
|
||||
data << r.ifexamine
|
||||
data << r.ifdisplay
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDealFJW(dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def stop = false
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def index = 0
|
||||
|
||||
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
|
||||
while(!stop){
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(index==1){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
if(curStartDateStr == curSndDateStr){
|
||||
stop = true
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
|
||||
def curEndStr = DateUtils.add(curStartStr, 1)
|
||||
curStartDateStr = curEndStr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataDealFJW2(curDate, dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
def exportAllBigdataDealFJW(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_DEAL_FJW 导出完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataBuildingFJW(cityCode){
|
||||
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.BUILDALIAS
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.FLOORCNT
|
||||
data << r.UNITCNT
|
||||
data << r.HOUSECNT
|
||||
data << r.IFELEVATOR
|
||||
data << r.ELEVATORCNT
|
||||
data << r.ELEVATORHOUSE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.VALID_FLAG
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def exportEvaluateRate(){
|
||||
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.RATE_TYPE
|
||||
data << r.RATE
|
||||
data << r.RANK
|
||||
data << r.CITY
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "EVALUATE_RATE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
/**
|
||||
* 重庆估值测试--2小时
|
||||
*/
|
||||
class EvaluationBatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new EvaluationBatch()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2021-01-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-07")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V1.xlsx", { row ->
|
||||
if(row['N'] && row['N'].toString().toDouble()>=20){
|
||||
println row['F'] +" "+row['N']
|
||||
def dist = CommonDatautils.findDistrictById(row['C'])
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48);
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def city = "重庆"
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row ->
|
||||
def region = row['E']
|
||||
def name = row['F']
|
||||
def id = row['C']
|
||||
def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30)
|
||||
def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365)
|
||||
if (region && name) {
|
||||
row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name)
|
||||
|
||||
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||
if(priceObj){
|
||||
row['AB'] = priceObj.avg_price
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(id)
|
||||
if(dist) {
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||
def map = getArroundAvgPriceMap(list)
|
||||
if(map){
|
||||
row['L'] = map.avg_price
|
||||
row['Q'] = map.max_val
|
||||
row['R'] = map.min_val
|
||||
}
|
||||
def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend)
|
||||
if(lastArroundPrice && priceObj){
|
||||
def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['P'] = mod
|
||||
}
|
||||
def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend)
|
||||
if(lastYearArroundPrice && priceObj){
|
||||
def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['O'] = mod
|
||||
}
|
||||
}
|
||||
def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend)
|
||||
if(lastWeekendObj){
|
||||
row['S'] = lastWeekendObj.avg_price
|
||||
if(priceObj){
|
||||
def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['M'] = mod
|
||||
}
|
||||
}
|
||||
def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend)
|
||||
if(lastYearObj){
|
||||
row['T'] = lastYearObj.avg_price
|
||||
if(priceObj){
|
||||
def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['N'] = mod
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "重庆"
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row ->
|
||||
def region = row['E']
|
||||
def name = row['F']
|
||||
def id = row['C']
|
||||
def area = row['G'].toDouble()
|
||||
if(region && name){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||
if(priceObj){
|
||||
row['J'] = priceObj.avg_price
|
||||
if(row['C']){
|
||||
row['I'] = (area*priceObj.avg_price).toInteger()
|
||||
}
|
||||
}
|
||||
def dist = CommonDatautils.findDistrictById(id)
|
||||
if(dist){
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||
row['N'] = getArroundAvgPrice(list)
|
||||
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
alias << dist.name
|
||||
def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10)
|
||||
if(houseAvgPrice){
|
||||
row['K'] = houseAvgPrice
|
||||
}
|
||||
|
||||
def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10)
|
||||
row['L'] = tradeAvgPrice
|
||||
|
||||
def avg_price = findAvgPrice(dist._id.toString())
|
||||
if(row['J'] && avg_price){
|
||||
def weekendPrice = row['J'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['P'] = ppe15
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, startArea, endArea){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
// q.put("source", "链家在线")
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
def price = null
|
||||
if(prices){
|
||||
price = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
|
||||
def getArroundAvgPriceMap(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
prices.sort()
|
||||
def map = [:]
|
||||
map.min_val = prices[0]
|
||||
map.max_val = prices[prices.size()-1]
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
map.avg_price = avg_price
|
||||
return map;
|
||||
}
|
||||
|
||||
def getArroundAvgPrice(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
|
||||
def getArroundAvgPriceByWeekend(def list, def weekend){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
def avg_price = null;
|
||||
if(prices){
|
||||
avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
|
||||
def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
def prices = []
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
def price = null
|
||||
if(prices){
|
||||
price = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
public String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class BlankMatch {
|
||||
static main(args) {
|
||||
def task = new BlankMatch()
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
}
|
||||
|
||||
def run() {
|
||||
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
|
||||
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['C']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['C'] = jsonObject.get("id")
|
||||
row['D'] = jsonObject.get("city")
|
||||
row['E'] = jsonObject.get("district")
|
||||
row['F'] = jsonObject.get("name")
|
||||
row['G'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1() {
|
||||
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row ->
|
||||
if(row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['C'] = jsonObject.get("id")
|
||||
row['D'] = jsonObject.get("city")
|
||||
row['J'] = jsonObject.get("name")
|
||||
}
|
||||
|
||||
}
|
||||
// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['C'] = jsonObject.get("id")
|
||||
// row['D'] = jsonObject.get("city")
|
||||
// row['E'] = jsonObject.get("district")
|
||||
// row['F'] = jsonObject.get("name")
|
||||
// row['G'] = jsonObject.get("address")
|
||||
// }
|
||||
});
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
|
||||
if(dist){
|
||||
row['C'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
for(def d : datas){
|
||||
if(d['B'] == row['B']){
|
||||
row['C'] = d['C']
|
||||
row['D'] = d['D']
|
||||
row['E'] = d['E']
|
||||
row['F'] = d['J']
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class FieldAddValue {
|
||||
static main(args) {
|
||||
def task = new FieldAddValue()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
//16991 14360
|
||||
def districts = CommonDatautils.findAllDistrictByCity("ÖØÇì", "district", ["region","name","alias","is_elevator"])
|
||||
def index = 0
|
||||
def updateIndex = 0
|
||||
for(def d : districts){
|
||||
if(!d.is_elevator){
|
||||
index++
|
||||
println index
|
||||
def alias = d.alias
|
||||
if(!d.alias){
|
||||
alias = []
|
||||
}
|
||||
alias << d.name
|
||||
def maxHeight = findHouseHeight("ÖØÇì", d.region, alias, 20220101)
|
||||
if(maxHeight && maxHeight>6){
|
||||
updateIndex++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("is_elevator", true)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId( d._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println index+" "+updateIndex
|
||||
}
|
||||
|
||||
def findHouseHeight(def city, def region, def alias, def startDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
def heights = null
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
if(data.height){
|
||||
heights = data.height
|
||||
}
|
||||
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
// def max_height = null
|
||||
// if(heights){
|
||||
// heights.sort()
|
||||
// max_height = heights[heights.size()-1]
|
||||
// }
|
||||
return heights
|
||||
}
|
||||
|
||||
public String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class MatchEvaluateRegionTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateRegionTest()
|
||||
task.run()//小区匹配
|
||||
task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
task.addPrice()
|
||||
task.addAuthor()
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("name")
|
||||
row['J'] = jsonObject.get("address")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("name")
|
||||
row['J'] = jsonObject.get("address")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['L']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
|
||||
if(priceObj){
|
||||
row['M'] = priceObj.avg_price
|
||||
row['N']= (int)(priceObj.avg_price * row['E'])
|
||||
}
|
||||
|
||||
def avg_price = findAvgPrice(row['M'])
|
||||
if(row['M'] && avg_price){
|
||||
def weekendPrice = row['M'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def total = 747
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(index < avg){
|
||||
row['P'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def flag = d['M']
|
||||
if(!flag){
|
||||
def city = d['E']
|
||||
def region = d['F']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['G']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['H']
|
||||
if(address){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
addPrice(data)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name.contains("(")){
|
||||
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
|
||||
name = name.replace(delstr,"")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
}
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
|
||||
class MatchEvaluateTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateTest()
|
||||
// task.run()//小区匹配
|
||||
// task.run1()//地图匹配
|
||||
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
// task.addPrice()
|
||||
// task.addArroundPrice()
|
||||
// task.addAuthor()
|
||||
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A'] && !row['K']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['K'] = priceObj.unitPrice
|
||||
row['L'] = (int) (priceObj.unitPrice * row['C'])
|
||||
row['O'] = "周边价格"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("address")
|
||||
row['I'] = jsonObject.get("propertyType")
|
||||
row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("address")
|
||||
row['I'] = jsonObject.get("propertyType")
|
||||
row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['J'] && row['E']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
|
||||
if(priceObj){
|
||||
row['K'] = priceObj.avg_price
|
||||
row['L']= (int)(priceObj.avg_price * row['C'])
|
||||
}
|
||||
|
||||
|
||||
def avg_price = findAvgPrice(row['D'])
|
||||
if(row['K'] && avg_price){
|
||||
def weekendPrice = row['K'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['B']){
|
||||
if(index < avg){
|
||||
row['N'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def flag = d['M']
|
||||
if(!flag){
|
||||
def city = d['E']
|
||||
def region = d['F']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['G']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['H']
|
||||
if(address){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
addPrice(data)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name.contains("(")){
|
||||
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
|
||||
name = name.replace(delstr,"")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(d['B']){
|
||||
for(def row : datas){
|
||||
if(d['B']==row['B']){
|
||||
if(row['K']){
|
||||
d['P'] = row['K']
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
+51
-1
@@ -218,7 +218,6 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","˝đťÔ˛ŽžôÉ˝")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -829,6 +828,9 @@ class CommonDatautils {
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
@@ -916,6 +918,23 @@ class CommonDatautils {
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static def findDistrictsByCityAndName(String city, String name, String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("cat", cat);
|
||||
q.put("name", name);
|
||||
q.put("visible", 0);
|
||||
|
||||
def list = []
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
list << obj;
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String name, String cat){
|
||||
@@ -3134,5 +3153,36 @@ class CommonDatautils {
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
static def findNewSqmetreRent(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1)
|
||||
def avg_price = null
|
||||
for(def d : dbcursor){
|
||||
avg_price = d.avg_price;
|
||||
}
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
static def findCityAvgRentRadio(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q)
|
||||
def avg_price = null
|
||||
def radios = []
|
||||
for(def d : dbcursor){
|
||||
if(d.price_to_rent){
|
||||
radios << d.price_to_rent.toInteger()
|
||||
}
|
||||
}
|
||||
def avg_radios = (int)(radios.sum()/radios.size())
|
||||
return avg_radios;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-3
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
|
||||
|
||||
class DataHandleUtils {
|
||||
|
||||
static main(args){
|
||||
|
||||
}
|
||||
|
||||
|
||||
static def getStandardRegion(def regionMap, def region){
|
||||
def newRegion = null;
|
||||
@@ -94,6 +92,8 @@ class DataHandleUtils {
|
||||
BigDecimal deSource = new BigDecimal(dSource);
|
||||
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def trimChar(String c) {
|
||||
try {
|
||||
|
||||
+5
-5
@@ -20,11 +20,11 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
task.regionPriceTrendRun()
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def map1 = [:]
|
||||
@@ -53,7 +53,7 @@ class FzDataResultStat {
|
||||
}
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
|
||||
}
|
||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
@@ -94,7 +94,7 @@ class FzDataResultStat {
|
||||
}
|
||||
def prices = exportRegion(city, months)
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
|
||||
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+5
-2
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
|
||||
retMap.anjuke_esf_price = d.anjuke_esf_price
|
||||
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("58tc_esf_price")
|
||||
}
|
||||
if(!retMap.soufang_esf_price && d.soufang_esf_price){
|
||||
retMap.soufang_esf_price = d.soufang_esf_price
|
||||
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
|
||||
retMap.lianjia_esf_price = d.lianjia_esf_price
|
||||
}
|
||||
if(!retMap.lianjia_esf_price && d.beike_esf_price){
|
||||
retMap.lianjia_esf_price = d.beike_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
|
||||
class ThirdTest {
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new ThirdTest()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.run3()
|
||||
// task.run4()
|
||||
|
||||
}
|
||||
|
||||
def run4(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
if (!d['C']) {
|
||||
// def name = d['A'].replace("(钢混)","").replace("(非钢混)","").replace("(混合)","")
|
||||
// def lists = CommonDatautils.findDistrictsByCityAndName("福州",name, "district")
|
||||
// if(lists){
|
||||
// for(def o : lists){
|
||||
//// if(d['B'] && d['B'].toString().contains(o.address)){
|
||||
// d['C'] = o.region
|
||||
// d['D'] = o.name
|
||||
// d['E'] = "名称去括号中的字"
|
||||
//// }
|
||||
// }
|
||||
// }
|
||||
def rMap = SeaweedDataMatchProcess.doMatch("福州", null, d['A'], d['B']);
|
||||
if(rMap){
|
||||
d['C'] = rMap.region
|
||||
d['D'] = rMap.name
|
||||
d['E'] = "匹配接口"
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run3() {
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
// if(!d['F']){
|
||||
// def price = CommonDatautils.queryDistrictPrice("福州",d['C'], d['D'])
|
||||
// d['F'] = price
|
||||
// }
|
||||
// },0)
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202202.csv"))
|
||||
// def map1 = [:]
|
||||
// for(def s : list1){
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
// map1.put(key, row[5])
|
||||
// }
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv"))
|
||||
def map1 = [:]
|
||||
for(def str : list1){
|
||||
def s = str.split("@\\|@")
|
||||
def key = s[2]+"##"+s[3]+"##"+s[4]
|
||||
def priceMap = [:]
|
||||
map1.put(key, priceMap)
|
||||
|
||||
if(s.size()>=7 && s[6]){
|
||||
priceMap.put("zhugezhaofang", s[6])
|
||||
}
|
||||
|
||||
if(s.size()>=8 && s[7]){
|
||||
priceMap.put("anjuke", s[7])
|
||||
}
|
||||
|
||||
if(s.size()>=9 && s[8]){
|
||||
priceMap.put("soufang", s[8])
|
||||
}
|
||||
|
||||
if(s.size()>=10 && s[9]){
|
||||
priceMap.put("lianjia", s[9])
|
||||
}
|
||||
|
||||
if(s.size()>=11 && s[10]){
|
||||
priceMap.put("cityhouse", s[10])
|
||||
}
|
||||
|
||||
if(s.size()>=12 && s[11]){
|
||||
priceMap.put("fangtan", s[11])
|
||||
}
|
||||
|
||||
if(s.size()>=13 && s[12]){
|
||||
priceMap.put("fangdoudou", s[12])
|
||||
}
|
||||
if(s.size()>=14 && s[13]){
|
||||
priceMap.put("maitian", s[13])
|
||||
}
|
||||
|
||||
try{
|
||||
if(s.size()>=15 && s[14]){
|
||||
priceMap.put("58tc", s[14])
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
// if (!d['F']) {
|
||||
// def lists = CommonDatautils.findDistrictsByCityAndName("福州", d['A'], "district")
|
||||
// if(lists){
|
||||
// for(def o : lists){
|
||||
// if(o.address==d['B']){
|
||||
// d['C'] = o.region
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
def key = "福州##"+d['C']+"##"+d['D']
|
||||
def priceMap = map1.get(key)
|
||||
if(priceMap){
|
||||
if(priceMap.zhugezhaofang){
|
||||
d['F'] = priceMap.zhugezhaofang
|
||||
}
|
||||
if(priceMap.anjuke){
|
||||
d['G'] = priceMap.anjuke
|
||||
}
|
||||
if(priceMap.soufang){
|
||||
d['H'] = priceMap.soufang
|
||||
}
|
||||
if(priceMap.lianjia){
|
||||
d['I'] = priceMap.lianjia
|
||||
}
|
||||
if(priceMap.cityhouse){
|
||||
d['J'] = priceMap.cityhouse
|
||||
}
|
||||
if(priceMap.fangtan){
|
||||
d['K'] = priceMap.fangtan
|
||||
}
|
||||
if(priceMap.fangdoudou){
|
||||
d['L'] = priceMap.fangdoudou
|
||||
}
|
||||
if(priceMap.maitian){
|
||||
d['M'] = priceMap.maitian
|
||||
}
|
||||
|
||||
if(priceMap.get("58tc")){
|
||||
d['N'] = priceMap.get("58tc")
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.modify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供-结果.xlsx",{d->
|
||||
if(d['B'] && !d['D']){
|
||||
if(d['B'].contains("号")){
|
||||
// def price = searchContent2("福州","https://fz.58.com", d['B'], 304)
|
||||
// if(price){
|
||||
// d['C'] = price
|
||||
// d['F'] = "58同城"
|
||||
// }
|
||||
def price = thirdPartyColl("福州", d['B'])
|
||||
if(price){
|
||||
d['D'] = price;
|
||||
d['F'] = "麦田"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def thirdPartyColl(def city, def address){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("address", address)
|
||||
q.put("source","麦田")
|
||||
q.put("s_date",202202)
|
||||
def obj = thirdPartyColl.findOne(q)
|
||||
if(obj){
|
||||
return obj.avg_price
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
//查看此关键字是否有数据
|
||||
def searchContent2(def city,def url,def keyword, def cityid){
|
||||
def newUrl = url+"/esf-ajax/community/pc/autocomplete/?city_id=${cityid}&kw="+URLEncoder.encode(keyword, "utf-8")+"&type=2"
|
||||
def document = CrawalUtil.getDocumentByABuYun(newUrl, "访问过于频繁", "utf-8", null, true);
|
||||
def price = null
|
||||
if(document){
|
||||
println "【"+keyword +"】 查询"
|
||||
def jsonObj = JSONObject.fromObject(document)
|
||||
if(jsonObj && jsonObj.status=="ok"){
|
||||
if(jsonObj.data.size()>0){
|
||||
def datas = jsonObj.data
|
||||
def d = datas[0]
|
||||
price = d.price.toInteger()
|
||||
}
|
||||
}else{
|
||||
println "【"+keyword +"】 无结果"
|
||||
}
|
||||
}
|
||||
return price;
|
||||
}
|
||||
|
||||
def run(){
|
||||
def map = findThirdPrice("福州")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供.xlsx",{d->
|
||||
if(!d['F']){
|
||||
def lists = CommonDatautils.findDistrictsByCityAndName("福州",d['A'], "district")
|
||||
if(!lists){
|
||||
lists = CommonDatautils.findDistrictByAlias("福州",null, d['A'], "district")
|
||||
}
|
||||
for(def o : lists){
|
||||
// if(o.address == d['B']){
|
||||
def priceMap = map.get(o.region+"##"+o.name)
|
||||
d['C'] = priceMap.anjuke_esf_price
|
||||
d['D'] = priceMap.maitian_esf_price
|
||||
d['E'] = priceMap.beike_esf_price
|
||||
d['F'] = o.region
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def findThirdPrice(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202109))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor) {
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.get("lianjia_esf_price") && db.lianjia_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ class EsHouseStat {
|
||||
|
||||
public static void main(String[] args){
|
||||
def task = new EsHouseStat();
|
||||
//task.run();
|
||||
task.run();
|
||||
// task.testRun("ÉîÛÚ")
|
||||
// task.exportOpenLog();
|
||||
// task.exportMongoLog()
|
||||
@@ -59,7 +59,7 @@ class EsHouseStat {
|
||||
// task.szRentHouseStat();
|
||||
// task.addPrice();
|
||||
|
||||
task.statNoHouseCCBData();
|
||||
// task.statNoHouseCCBData();
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
@@ -337,21 +337,6 @@ class EsHouseStat {
|
||||
return total;
|
||||
}
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
def findHouseSource(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
|
||||
@@ -373,6 +358,21 @@ class EsHouseStat {
|
||||
return statMap;
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
def testRun(def city){
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
|
||||
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","")
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
@@ -149,6 +149,7 @@ class OpenApiUtils {
|
||||
ret.lng2 = gmap.get("lng")
|
||||
ret.lat2 = gmap.get("lat")
|
||||
ret.estateType = ret.type
|
||||
ret.source = "高德"
|
||||
}catch (Exception e){
|
||||
|
||||
}
|
||||
@@ -250,14 +251,14 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
if(type && type.contains("出入口")){
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
}else{
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
poiList.add(jo);
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
@@ -302,7 +303,7 @@ class OpenApiUtils {
|
||||
def distname = formatName(ret.get("name"))
|
||||
address = ret.get("address")
|
||||
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
|
||||
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
lng2 = location.lng
|
||||
lat2 = location.lat
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
@@ -311,7 +312,7 @@ class OpenApiUtils {
|
||||
def mapRegion = ret.get("area")
|
||||
distRegion = mapRegion
|
||||
districtName = distname
|
||||
if(address && address.contains(mapRegion)){
|
||||
if(address && mapRegion && address.contains(mapRegion)){
|
||||
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
|
||||
}
|
||||
if(ret.get("detail_info")){
|
||||
@@ -330,6 +331,7 @@ class OpenApiUtils {
|
||||
map.name = districtName
|
||||
map.region = distRegion
|
||||
map.estateType = estateType
|
||||
map.source = "百度"
|
||||
rets << map
|
||||
|
||||
lat = null
|
||||
@@ -340,12 +342,18 @@ class OpenApiUtils {
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
// }else{
|
||||
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
// }
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
def score1 = 0
|
||||
if(distname && name){
|
||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
}
|
||||
def score2 = 0
|
||||
if(address && name){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
}
|
||||
if(score1 || score2){
|
||||
rets << map
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
|
||||
|
||||
@@ -31,7 +31,7 @@ class RegionPointCrawl {
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
|
||||
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.qqmap.TengXunApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
@@ -42,4 +43,29 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"
|
||||
// }
|
||||
//
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
|
||||
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
|
||||
if(d['B']){
|
||||
def json = GaoDeApiUtils.geo("重庆", d['B'])
|
||||
if(json && json.info=="OK"){
|
||||
def datas = json.geocodes
|
||||
if(datas){
|
||||
def row = datas[0]
|
||||
println row
|
||||
d['C'] = row.country
|
||||
d['D'] = row.province
|
||||
d['E'] = row.city.toString().replace("[]","")
|
||||
d['F'] = row.citycode.toString().replace("[]","")
|
||||
d['G'] = row.district.toString().replace("[]","")
|
||||
d['H'] = row.street.toString().replace("[]","")
|
||||
d['I'] = row.number.toString().replace("[]","")
|
||||
d['J'] = row.adcode.toString().replace("[]","")
|
||||
d['K'] = row.location
|
||||
d['L'] = row.level
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
@@ -7,11 +7,11 @@ email.setSSL(true);
|
||||
email.setHostName("smtp.exmail.qq.com");
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
|
||||
email.addTo("gaojuan@fangjia.com"); //接收方
|
||||
email.setFrom("service@fangjia.com"); //发送方
|
||||
email.setSubject("test"); //标题
|
||||
email.setContent("你?", "text/html");
|
||||
email.setContent("测试", "text/html");
|
||||
|
||||
//email.setHostName("smtp.exmail.qq.com");
|
||||
//email.setSSL(true);
|
||||
|
||||
+42
-25
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
/**
|
||||
* 模糊地址经纬度更新
|
||||
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
//"北京","常州",
|
||||
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
||||
def citys =["上海"] //CommonDatautils.findAllCity();
|
||||
|
||||
def citys =["重庆"] //CommonDatautils.findAllCity();
|
||||
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
||||
//市谊大厦 华三新村 阳光家园
|
||||
for (def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
|
||||
def rets = findVagueAddress(city, districts);
|
||||
findAndUpdateLocation(rets);
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
|
||||
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
|
||||
def findVagueAddress(def city, def districts) {
|
||||
def rets = []
|
||||
def rows = []
|
||||
for (def district : districts) {
|
||||
def address = district.address;
|
||||
if (address) {
|
||||
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
//没地址
|
||||
if (!address) {
|
||||
rets << district;
|
||||
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
if (!findPattern(address)) {
|
||||
rets << district;
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
println district.city + " " + district.address + " 精确地址"
|
||||
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
|
||||
return null;
|
||||
} else {
|
||||
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
|
||||
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
|
||||
matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return s;
|
||||
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
for (def ret : rets) {
|
||||
index++;
|
||||
|
||||
if(ret.name!="药师新村"){
|
||||
continue
|
||||
}
|
||||
|
||||
def locationMap = findLatAndLng(ret);
|
||||
if (locationMap) {
|
||||
updateLocation(locationMap, ret);
|
||||
} else {
|
||||
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
|
||||
}
|
||||
println "【${index}/${rets.size()}】************************************************************************************"
|
||||
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
|
||||
|
||||
}
|
||||
}
|
||||
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
msgMap.db_lng2 = ret.lng2
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
|
||||
if (ret.lat2 != locationMap.lat2) {
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
|
||||
msgMap.m_lat2 = locationMap.lat2
|
||||
msgMap.m_lng2 = locationMap.lng2
|
||||
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
|
||||
msgMap.updateLngFlag = true
|
||||
msgMap.distance = distance
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
}
|
||||
|
||||
if (ret.lat != locationMap.lat) {
|
||||
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
if (udata) {
|
||||
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
|
||||
udata.put("m_date", new Date())
|
||||
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
results << msgMap
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def name, def rets) {
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() > 1) {
|
||||
//相似里找有精确地址
|
||||
for (def mapObj : rets) {
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!map){
|
||||
//交带有交叉口地址
|
||||
for (def mapObj : rets) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
|
||||
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
|
||||
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
def findLatAndLng(ret) {
|
||||
//https://restapi.amap.com/v3/place/text?keywords=
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
|
||||
def map = findStandardObject(ret.city, ret.name, rets);
|
||||
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
//http://api.map.baidu.com/place/v2/search
|
||||
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
if (!map || !map.lat) {
|
||||
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
|
||||
def code = codeMap.get(ret.city);
|
||||
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
|
||||
// if((!map || !map.lat)){
|
||||
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
|
||||
locationMap.id = ret._id
|
||||
locationMap.city = ret.city
|
||||
if (ret.address.toString().endsWith("路")) {
|
||||
if (map.address) {
|
||||
locationMap.address = map.address.replace("(","").replace(")","");
|
||||
}
|
||||
|
||||
|
||||
+8
-7
@@ -37,22 +37,22 @@ class YouDaDataParser {
|
||||
public static final DBCollection youda_newhouse_coll = MongoUtils.getCollectionByName2("business", "res_newdealprice", offlineUrl)
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
//线下:228 线上:45
|
||||
//线下:228 线上:45 测试
|
||||
static def new_targetUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc"
|
||||
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
|
||||
|
||||
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-12-01"
|
||||
static def statDate = "2022-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
@@ -490,8 +490,9 @@ class YouDaDataParser {
|
||||
if(s){
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
if(hits && hits.total instanceof Integer){
|
||||
total = hits.total
|
||||
|
||||
}
|
||||
}
|
||||
return total
|
||||
@@ -565,7 +566,7 @@ class YouDaDataParser {
|
||||
if(obj.premisesName && obj.premisesName!="null"){
|
||||
dbo.premisesName = obj.premisesName
|
||||
}
|
||||
dbo.price = obj.price
|
||||
dbo.price = obj.price.toInteger()
|
||||
dbo.region = obj.region
|
||||
dbo.resId = obj.ydId.toString()
|
||||
dbo.totalPrice = obj.totalPrice
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.alibaba.excel.util.DateUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.Calendar;
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiTimeOutAnalyBatch extends BaseBatch{
|
||||
|
||||
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiTimeOutAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def uriCountMap = findTimeOutUri()
|
||||
sendMessage(uriCountMap)
|
||||
}
|
||||
|
||||
def sendMessage(uriCountMap){
|
||||
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
|
||||
def total = message.length()
|
||||
uriCountMap.each{uri,count->
|
||||
if(count>=5){
|
||||
message = message + uri +"\n"
|
||||
}
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
}
|
||||
|
||||
def findTimeOutUri(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("time", new BasicDBObject('$gte', 3000))
|
||||
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
|
||||
q.put("c_date", new BasicDBObject('$gte', startTime))
|
||||
def dbcursor = LOG.find(q)
|
||||
def uriCountMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def count = uriCountMap.get(d.request_uri)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
uriCountMap.put(d.request_uri, count + 1)
|
||||
}
|
||||
return uriCountMap
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiUserCountAnalyBatch extends BaseBatch{
|
||||
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
|
||||
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiUserCountAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def rets = findAllUsers()
|
||||
sendMessage(rets)
|
||||
}
|
||||
|
||||
|
||||
def sendMessage(def rets){
|
||||
def message = "【Api接口使用提醒】\n"
|
||||
def total = message.length()
|
||||
for(def r : rets){
|
||||
def type = "正式帐户"
|
||||
if(r.type!=0){
|
||||
type = "测试帐户"
|
||||
}
|
||||
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllUsers(){
|
||||
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
|
||||
def rows = dbFangjia.rows(sql)
|
||||
def rets = []
|
||||
for(def row : rows){
|
||||
def obj = findResources(row.id)
|
||||
if(obj){
|
||||
def map = obj.total_count_map
|
||||
def uris = []
|
||||
map.each{k,v->
|
||||
//def visit_count = findVisitCount(row.id, k)
|
||||
def total_visit_count = findAllVisitCount(row.id, k)
|
||||
println row.id +" "+k+" "+v+" "+total_visit_count
|
||||
if(v && total_visit_count){
|
||||
def precent = v*0.8;
|
||||
if(total_visit_count>=precent){
|
||||
uris << k
|
||||
}
|
||||
}
|
||||
}
|
||||
if(uris){
|
||||
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
|
||||
def dbcursor = VISIT_COLL.find(q)
|
||||
def total = 0
|
||||
for(def obj : dbcursor){
|
||||
total = total +obj.visit_count
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", DateUtils.truncateDay(new Date()))
|
||||
def obj = VISIT_COLL.findOne(q)
|
||||
if(obj){
|
||||
return obj.visit_count
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findResources(def userId){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId)
|
||||
return AUTHORITY_COLL.findOne(q)
|
||||
}
|
||||
}
|
||||
+8
-5
@@ -62,12 +62,12 @@ class CcbCodeUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
// 高层塔楼 0101 增加“超高层” 0108
|
||||
// 高层板楼 0102 增加“老洋房” 0109
|
||||
// 高层塔楼 0101 增加“超高层” 0109
|
||||
// 高层板楼 0102 增加“老洋房” 0113
|
||||
// 小高层板楼 0103 增加“花园洋房” 0110
|
||||
// 多层板楼 0104 增加“四合院” 0111
|
||||
// 独栋别墅 0105 增加“老式别墅洋房” 0112
|
||||
// 联排别墅 0106
|
||||
// 联排别墅 0106 增加“双拼别墅” 0108
|
||||
// 叠拼别墅 0107
|
||||
// 其他建筑形式 0199
|
||||
|
||||
@@ -84,7 +84,10 @@ class CcbCodeUtils {
|
||||
types << "0110"
|
||||
}
|
||||
if(buildType.contains("老洋房")){
|
||||
types << "0109"
|
||||
types << "0113"
|
||||
}
|
||||
if(buildType.contains("双拼")){
|
||||
types << "0108"
|
||||
}
|
||||
if(buildType.contains("叠拼")){
|
||||
types << "0107"
|
||||
@@ -99,7 +102,7 @@ class CcbCodeUtils {
|
||||
types << "0104"
|
||||
}
|
||||
if(buildType.contains("超高层")){
|
||||
types << "0108"
|
||||
types << "0109"
|
||||
}
|
||||
if(buildType.contains("小高层")){
|
||||
types << "0103"
|
||||
|
||||
+2
-2
@@ -25,11 +25,11 @@ class DataMatch {
|
||||
static main(args) {
|
||||
def task = new DataMatch()
|
||||
// task.run1()
|
||||
task.run2()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
|
||||
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
|
||||
class DistrictSelectHouse {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSelectHouse();
|
||||
// task.run()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本V1 -跑地址.xlsx",{d->
|
||||
if(d['A']) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'],d['B'], d['C'], "district")
|
||||
if(dist){
|
||||
d['L'] = dist.address
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本.xlsx",{d->
|
||||
if(d['N']){
|
||||
def floorInfo = d['J']
|
||||
def height = d['N']
|
||||
if(floorInfo && height){
|
||||
height = height.toInteger()
|
||||
def mod = (int)(height/3)
|
||||
def h_start = (height - mod).toInteger()
|
||||
def random_floor = null
|
||||
if(floorInfo.contains("低楼层")){
|
||||
random_floor = random(1, mod)
|
||||
}else if(floorInfo.contains("中楼层")){
|
||||
random_floor = random(mod+1, h_start-1)
|
||||
}else if(floorInfo.contains("高楼层")){
|
||||
random_floor = random(h_start, height)
|
||||
}
|
||||
d['M'] = random_floor
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
|
||||
int random(int i1, int i2) {
|
||||
int max = Math.max(i1, i2);
|
||||
int min = Math.min(i1, i2);
|
||||
return min + RandomUtils.nextInt(max - min + 1);
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "重庆"
|
||||
def weekend = "2022-03-06"
|
||||
def thirdPriceMap = findNearThirdPartyPrice(city)
|
||||
def distStats = findCreditHightDistrict2(city, weekend, thirdPriceMap)
|
||||
findAllHouse(distStats);
|
||||
}
|
||||
|
||||
def findAllHouse(distStats){
|
||||
def houses = []
|
||||
distStats.each{k,v->
|
||||
def arrs = k.split("#")
|
||||
def list = findHouses(arrs[0],arrs[1], arrs[2], 20220201,20220401)
|
||||
if(list){
|
||||
houses.addAll(list)
|
||||
}
|
||||
}
|
||||
if(houses){
|
||||
def rets = []
|
||||
for(def h : houses){
|
||||
def row = []
|
||||
row << h.city
|
||||
row << h.region
|
||||
row << h.district_name
|
||||
row << h.area
|
||||
row << h.avg_price
|
||||
row << h.total_price
|
||||
row << h.direction
|
||||
row << h.estate_type
|
||||
row << h.fitment
|
||||
row << h.floor_info
|
||||
row << h.room
|
||||
row << h.hall
|
||||
row << h.floor
|
||||
row << h.height
|
||||
row << h.p_date
|
||||
rets << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆样本.xlsx", rets, "数据")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def startDate, def endDate){
|
||||
String sql = "select * from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and region='${region}' and district_name='${name}' and p_date >= ${startDate} and p_date<${endDate}"
|
||||
def houses = []
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
houses << data
|
||||
})
|
||||
if(houses.size()>10){
|
||||
houses = houses[0..9]
|
||||
}
|
||||
return houses
|
||||
}
|
||||
|
||||
|
||||
def findNearThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202108))
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def thirdPriceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.region+"#"+d.name
|
||||
if(!thirdPriceMap.containsKey(key)){
|
||||
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return thirdPriceMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def weekend, def thirdPriceMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
|
||||
if(thirdPrice) {
|
||||
def precent = (thirdPrice - d.avg_price) * 100 / d.avg_price
|
||||
if (Math.abs(precent) >= 0 && Math.abs(precent) <= 10) {
|
||||
retMaps.put(d.city +"#"+d.region+"#"+d.district_name, d.avg_price)
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMaps
|
||||
}
|
||||
|
||||
}
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeDetailCrawl {
|
||||
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
static main(args) {
|
||||
def task = new AnjukeDetailCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://chongqing.anjuke.com/community/?from=navigation"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def regions = getRegions(driver, url)
|
||||
getAllRegions(driver, regions)
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getAllRegions(driver, regions){
|
||||
for(def url : regions){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def total = getTotal(doc)
|
||||
if(total) {
|
||||
if (total > 1250) {
|
||||
|
||||
} else {
|
||||
def links = getAllLinks(url, driver)
|
||||
parse(links,driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getAllLinks(url, driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
while(!stop) {
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def details = JQueryUtils.select(".list-cell a.li-row", doc)
|
||||
|
||||
for (def d : details) {
|
||||
urls << d.attr("href")
|
||||
}
|
||||
def nextEle = JQueryUtils.select(".next", doc)
|
||||
if (nextEle) {
|
||||
driver.findElement(By.cssSelector(".next")).click()
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
if(urls.size()>20){
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
def getTotal(doc){
|
||||
|
||||
def totalText = JQueryUtils.selectText(".total-info", doc)
|
||||
if(totalText){
|
||||
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
|
||||
if(total && total.isNumber()){
|
||||
return total.toInteger()
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getRegions(driver, url){
|
||||
driver.get(url)
|
||||
Thread.sleep(3000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
|
||||
// if(loginIfm){
|
||||
// def href = loginIfm[0].attr("src")
|
||||
// driver.get(href)
|
||||
// driver.findElementById("pwdTab").click()
|
||||
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
|
||||
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
|
||||
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
|
||||
// elemUsername.sendKeys("17740818975");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
// btn.submit(); // 点击行为;
|
||||
//
|
||||
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
// def urls = []
|
||||
// for(def e : regionEles){
|
||||
// urls << e.attr("href")
|
||||
// }
|
||||
// return urls;
|
||||
// }
|
||||
def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
def urls = []
|
||||
for(def e : regionEles){
|
||||
urls << e.attr("href")
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def parse(def links, driver){
|
||||
def rets = []
|
||||
for(def link : links){
|
||||
def district = parsing(link, driver)
|
||||
if(district){
|
||||
rets << district
|
||||
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
}
|
||||
if (rets.size()> 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = detailColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
detailColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
detailColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
|
||||
|
||||
|
||||
def parsing(url, driver){
|
||||
|
||||
def district = [:]
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def headers = JQueryUtils.select(".crumbs a", doc)
|
||||
if(headers){
|
||||
if (headers && headers.size() >= 3) {
|
||||
district.city = headers[1].text().replace("小区", "")
|
||||
district.region = headers[2].text().replace("小区", "")
|
||||
district.name = headers[headers.size() - 1].text().replace("二手房", "")
|
||||
}
|
||||
|
||||
|
||||
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
|
||||
|
||||
def priceStr = JQueryUtils.selectText(".average", doc)
|
||||
|
||||
if (priceStr && priceStr.isNumber()) {
|
||||
district.avg_price = priceStr.toInteger()
|
||||
}
|
||||
|
||||
def infoDiv = JQueryUtils.select(".info-list", doc)
|
||||
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
|
||||
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
||||
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
||||
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
||||
|
||||
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
|
||||
if (houseCountStr) {
|
||||
houseCountStr = houseCountStr.replace("户", "")
|
||||
if (houseCountStr) {
|
||||
district.house_count = houseCountStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
||||
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
|
||||
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
|
||||
if(carSiteStr){
|
||||
if(carSiteStr.contains("(")){
|
||||
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
|
||||
}
|
||||
if(carSiteStr && carSiteStr.isNumber()){
|
||||
district.car_site = carSiteStr
|
||||
}
|
||||
}
|
||||
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
||||
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
|
||||
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
||||
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
||||
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
||||
district.url = url
|
||||
district.c_date = new Date()
|
||||
district.m_date = new Date()
|
||||
district.root = "anjuke.com"
|
||||
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(3000)
|
||||
count++
|
||||
}
|
||||
if(count>10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
return district
|
||||
}
|
||||
|
||||
String getValue(String s){
|
||||
if(s && s.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class CommonCrawl {
|
||||
static def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
}
|
||||
+77
-52
@@ -55,7 +55,7 @@ class AddDistrictFieldValue {
|
||||
// }
|
||||
// })
|
||||
|
||||
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
|
||||
def citys = ["重庆"]//CcbCommonUtils.citys;
|
||||
//第三方均价数据匹配
|
||||
// for(def city : citys){
|
||||
// def rets = findAllDataByCity(city)
|
||||
@@ -63,11 +63,11 @@ class AddDistrictFieldValue {
|
||||
// }
|
||||
|
||||
//第三方均价数据填补
|
||||
// for(def city : citys){
|
||||
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
// def rMap = getMatchDistrict(city)
|
||||
// createResultValue(dists, rMap)
|
||||
// }
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
def rMap = getMatchDistrict(city)
|
||||
createResultValue(dists, rMap)
|
||||
}
|
||||
|
||||
// for(def city : citys){
|
||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
|
||||
@@ -103,52 +103,52 @@ class AddDistrictFieldValue {
|
||||
// }
|
||||
|
||||
|
||||
def citys = ["上海"]
|
||||
def updateIndex = 0
|
||||
def rets = []
|
||||
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
|
||||
def total = dists.size()
|
||||
|
||||
for(def dist : dists){
|
||||
if(dist.household_count){
|
||||
def ret = []
|
||||
ret << dist._id.toString()
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.household_count
|
||||
ret << dist.loudong_count
|
||||
def details = findDistrictDetail(dist)
|
||||
if(details){
|
||||
for(def d : details){
|
||||
def region = RegionData.formatRegion(city, d.region)
|
||||
if(d.region == region){
|
||||
if(dist.household_count && dist.household_count==d.house_count){
|
||||
ret << d.loudong_count
|
||||
ret << "链家"
|
||||
// updateIndex++
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("household_count", d.house_count.toInteger())
|
||||
// udata.put("m_date", new Date())
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
|
||||
}
|
||||
// def citys = ["上海"]
|
||||
// def updateIndex = 0
|
||||
// def rets = []
|
||||
// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
|
||||
// for(def city : citys){
|
||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
|
||||
// def total = dists.size()
|
||||
//
|
||||
// for(def dist : dists){
|
||||
// if(dist.household_count){
|
||||
// def ret = []
|
||||
// ret << dist._id.toString()
|
||||
// ret << dist.city
|
||||
// ret << dist.region
|
||||
// ret << dist.name
|
||||
// ret << dist.household_count
|
||||
// ret << dist.loudong_count
|
||||
// def details = findDistrictDetail(dist)
|
||||
// if(details){
|
||||
// for(def d : details){
|
||||
// def region = RegionData.formatRegion(city, d.region)
|
||||
// if(d.region == region){
|
||||
// if(dist.household_count && dist.household_count==d.house_count){
|
||||
// ret << d.loudong_count
|
||||
// ret << "链家"
|
||||
//// updateIndex++
|
||||
//// def udata = new BasicDBObject()
|
||||
//// udata.put("household_count", d.house_count.toInteger())
|
||||
//// udata.put("m_date", new Date())
|
||||
//// def q = new BasicDBObject()
|
||||
//// q.put("_id", dist._id)
|
||||
//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
|
||||
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }else{
|
||||
// ret << ""
|
||||
// ret << ""
|
||||
// }
|
||||
//
|
||||
// rets << ret
|
||||
// }
|
||||
// }
|
||||
// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
@@ -303,6 +303,14 @@ class AddDistrictFieldValue {
|
||||
def createResultValue(dists, rMap){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
def addressIndex = 0
|
||||
def allAreaIndex = 0
|
||||
def buildAreaIndex = 0
|
||||
def virescence_percent_index = 0
|
||||
def cubage_percent_index = 0
|
||||
def finish_building_date_index = 0
|
||||
def household_count_index = 0
|
||||
def car_site_count_index = 0
|
||||
for(def d : dists){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
@@ -311,12 +319,14 @@ class AddDistrictFieldValue {
|
||||
if(!d.address){
|
||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||
if(lastValue){
|
||||
addressIndex++
|
||||
udata.put("address", lastValue)
|
||||
}
|
||||
}
|
||||
if(!d.all_area){
|
||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
allAreaIndex++
|
||||
udata.put("all_area", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -324,6 +334,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.build_area){
|
||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
buildAreaIndex++
|
||||
udata.put("build_area", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -331,6 +342,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.virescence_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
virescence_percent_index ++
|
||||
udata.put("virescence_percent", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -338,6 +350,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.cubage_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
cubage_percent_index ++
|
||||
udata.put("cubage_percent", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -345,6 +358,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.finish_building_date){
|
||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||
if(lastValue){
|
||||
finish_building_date_index ++
|
||||
udata.put("finish_building_date", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -352,6 +366,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.household_count){
|
||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||
if(lastValue){
|
||||
household_count_index ++
|
||||
udata.put("household_count", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -359,6 +374,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.car_site_count){
|
||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||
if(lastValue){
|
||||
car_site_count_index ++
|
||||
udata.put("car_site_count", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -373,6 +389,15 @@ class AddDistrictFieldValue {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "addressIndex "+ addressIndex
|
||||
println "allAreaIndex "+ allAreaIndex
|
||||
println "buildAreaIndex "+ buildAreaIndex
|
||||
println "virescence_percent_index "+ virescence_percent_index
|
||||
println "cubage_percent_index "+ cubage_percent_index
|
||||
println "finish_building_date_index "+ finish_building_date_index
|
||||
println "household_count_index "+ household_count_index
|
||||
println "car_site_count_index "+ car_site_count_index
|
||||
}
|
||||
|
||||
def getValuesByField(sourceMap, field, estateType){
|
||||
|
||||
+14
-2
@@ -12,7 +12,19 @@ class TempMatchDistrict {
|
||||
static main(args) {
|
||||
def task = new TempMatchDistrict()
|
||||
// task.run()
|
||||
task.regionConver()
|
||||
// task.regionConver()
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def total = 0
|
||||
for(def city : citys){
|
||||
def count = CommonDatautils.findAllDistrictCount("district", city)
|
||||
total = total + count
|
||||
println city +" "+count
|
||||
}
|
||||
println "共:"+total
|
||||
}
|
||||
|
||||
def regionConver(){
|
||||
@@ -29,7 +41,7 @@ class TempMatchDistrict {
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
|
||||
def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"]
|
||||
def files = ["杭州.xlsx"]
|
||||
for(def f : files){
|
||||
def filePath = path+"\\"+f;
|
||||
match(filePath);
|
||||
|
||||
+20
-3
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
@@ -38,8 +38,8 @@ class FangjiaClientUtils {
|
||||
}
|
||||
|
||||
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
def content = null;
|
||||
try{
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class ZsNoExistsToDB {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new ZsNoExistsToDB()
|
||||
task.run()
|
||||
// task.addDistrictToDB()
|
||||
}
|
||||
|
||||
def addDistrictToDB(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
|
||||
def keys = []
|
||||
def addIndex = 0
|
||||
for(def d : datas){
|
||||
if(d['J'] && d['J']=="新增"){
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def name = d['F'].toString().replace("·","")
|
||||
def address = d['G']
|
||||
def dist = CommonDatautils.findDistrict(city,region,name,"district")
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
|
||||
}
|
||||
if(!dist){
|
||||
keys << city+"##"+region+"##"+name
|
||||
def alias = []
|
||||
def str = d['H']
|
||||
if(str){
|
||||
def arrs = str.split(",")
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address){
|
||||
address = AddressUtil.formatAddress(city, address)
|
||||
}
|
||||
def lng = d['M']
|
||||
def lat = d['L']
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
long lng_new = 0
|
||||
long lat_new = 0
|
||||
long lng2_new = 0
|
||||
long lat2_new = 0
|
||||
if(lng){
|
||||
lng = lng.toString().toDouble()
|
||||
lat = lat.toString().toDouble()
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
lng2 = baidu.x
|
||||
lat2 = baidu.y
|
||||
lng_new = lng *10000000000L
|
||||
lat_new = lat *10000000000L
|
||||
if(lng2){
|
||||
lng2_new = lng2 *10000000000L
|
||||
lat2_new = lat2 *10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", address)
|
||||
if(lng_new){
|
||||
data.put("lat",lat_new)
|
||||
data.put("lng",lng_new)
|
||||
data.put("lat2",lat2_new)
|
||||
data.put("lng2",lng2_new)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(data)
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
|
||||
addIndex++
|
||||
println "[新增] "+addIndex +" "+ data
|
||||
|
||||
}else{
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findOpenLog2()
|
||||
// ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
|
||||
println rets.size()
|
||||
|
||||
// matchMap2()
|
||||
}
|
||||
|
||||
def matchMap2() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
|
||||
if(!d['E']){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
}
|
||||
if(map){
|
||||
println map
|
||||
if(map.adname){
|
||||
d['E'] = map.adname
|
||||
}else if(map.region){
|
||||
d['E'] = map.region
|
||||
}
|
||||
d['F'] = map.name
|
||||
d['G'] = map.address
|
||||
d['K'] = map.estateType
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.lng
|
||||
d['N'] = map.lat2
|
||||
d['O'] = map.lng2
|
||||
d['P'] = map.source
|
||||
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def matchMap(){
|
||||
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
|
||||
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
|
||||
if(!gdData){
|
||||
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
|
||||
}
|
||||
if(gdData){
|
||||
data['E'] = gdData.district
|
||||
data['F'] = gdData.name
|
||||
data['G'] = gdData.address
|
||||
data['H'] = gdData.type
|
||||
data['I'] = gdData.lng
|
||||
data['J'] = gdData.lat
|
||||
data['K'] = gdData.score
|
||||
}
|
||||
|
||||
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
|
||||
if(!bdData){
|
||||
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
|
||||
}
|
||||
if(bdData){
|
||||
data['L'] = bdData.district
|
||||
data['M'] = bdData.name
|
||||
data['N'] = bdData.address
|
||||
data['O'] = bdData.type
|
||||
data['P'] = bdData.lng
|
||||
data['Q'] = bdData.lat
|
||||
data['R'] = bdData.score
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findOpenLog2(){
|
||||
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
|
||||
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def index = 0
|
||||
def count = 0
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d.code == 400){
|
||||
index++
|
||||
def msg = d.msg
|
||||
if(msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def params = d.parameter_map
|
||||
def address = null
|
||||
if(params && params.address){
|
||||
address = params.address
|
||||
}
|
||||
def r = [rMap.city, rMap.region, rMap.name, address]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findOpenLog(){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
|
||||
sqlBuffer.append(" and user_id = '1007569'")
|
||||
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
|
||||
sqlBuffer.append(" and c_date >= '2022-01-01'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def index = 0
|
||||
def count = 0
|
||||
while (rs.next()){
|
||||
def code = rs.getInt("code")
|
||||
if(code == 400){
|
||||
index++
|
||||
def msg = rs.getString("msg")
|
||||
if(msg && msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def params = rs.getObject("parameter_map")
|
||||
println params
|
||||
def r = [rMap.city, rMap.region, rMap.name]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+307
-44
@@ -1,15 +1,18 @@
|
||||
package com.fangjia.script.gj.level
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
@@ -18,7 +21,11 @@ import groovy.sql.Sql
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.util.DateUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
@@ -44,13 +51,15 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc"
|
||||
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictIndexBatch(), args);
|
||||
}
|
||||
@@ -63,9 +72,8 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
def run(){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
citys.remove("上海")
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
|
||||
def code = cityCodeDatas.get(city)
|
||||
def city_avg_total_price = null
|
||||
if(city!="ÉϺ£"){
|
||||
@@ -78,7 +86,9 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
city_avg_total_price = 531.2615913
|
||||
}
|
||||
if(city_avg_total_price){
|
||||
calcute(city, dists, code, city_avg_total_price)
|
||||
// calcute(city, dists, code, city_avg_total_price)
|
||||
calcuteIndex(city, dists, code, city_avg_total_price)
|
||||
|
||||
}
|
||||
println city +" "+city_avg_total_price
|
||||
}
|
||||
@@ -110,48 +120,28 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
def total_price = getHousePrice(d, dists, priceMap, citycode);
|
||||
def avg_cost = getAvgCost(d, costDatas)
|
||||
|
||||
def total_price_index = 0
|
||||
if (total_price) {
|
||||
total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508
|
||||
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
|
||||
}
|
||||
def avg_charge_index = 0
|
||||
if (avg_charge) {
|
||||
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
|
||||
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
|
||||
}
|
||||
|
||||
def rank_index = 0
|
||||
if (rank) {
|
||||
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
|
||||
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
|
||||
}
|
||||
|
||||
def avg_cost_index = 0
|
||||
if (avg_cost) {
|
||||
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
|
||||
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
|
||||
}
|
||||
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
|
||||
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
|
||||
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
def map = [:]
|
||||
map.id = d._id.toString()
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.name
|
||||
map.avg_total_price = total_price_index
|
||||
|
||||
map.avg_estate_charge = avg_charge
|
||||
map.price_rank = rank
|
||||
map.avg_cost = avg_cost
|
||||
map.comp_value = potentiality
|
||||
map.month = "2022-01"
|
||||
map.total_price = total_price
|
||||
//map.comp_value = potentiality
|
||||
//map.avg_total_price = total_price_index
|
||||
map.month = month
|
||||
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
|
||||
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
|
||||
|
||||
def newUrl = targetUrl +"/"+map.id
|
||||
map.sell_avg_price = avg_price
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
|
||||
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price+""
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -163,16 +153,288 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
latch.await();
|
||||
|
||||
|
||||
def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
|
||||
def fileName = "/home/gj/potentiality_${city}.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
|
||||
// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
|
||||
// def fileName = "/home/gj/potentiality_${city}.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
|
||||
// }
|
||||
// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
|
||||
// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
||||
}
|
||||
|
||||
def calcuteIndex(city, dists, citycode, city_avg_total_price){
|
||||
def detailMap = findAllDistrictDatas(city)
|
||||
def avgMap = calcuteAvgValues(detailMap);
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def avg_total_price = avgMap.avg_price
|
||||
def avg_charge = avgMap.avg_charge
|
||||
def avg_cost = avgMap.avg_cost
|
||||
def avg_rank = avgMap.avg_rank
|
||||
|
||||
def sd_avg_price = avgMap.sd_avg_price
|
||||
def sd_avg_charge = avgMap.sd_avg_charge
|
||||
def sd_avg_cost = avgMap.sd_avg_cost
|
||||
def sd_avg_rank = avgMap.sd_avg_rank
|
||||
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def rMap = detailMap.get(d._id.toString())
|
||||
if(rMap){
|
||||
def total_price_index = 0
|
||||
if (rMap.total_price) {
|
||||
// total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / 313.3693508
|
||||
total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / sd_avg_price
|
||||
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
|
||||
}
|
||||
def avg_charge_index = 0
|
||||
if (rMap.avg_estate_charge) {
|
||||
// avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / 0.772635241
|
||||
avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / sd_avg_charge
|
||||
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
|
||||
}
|
||||
|
||||
def rank_index = 0
|
||||
if (rMap.price_rank) {
|
||||
// rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / 0.261907636
|
||||
rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / sd_avg_rank
|
||||
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
|
||||
}
|
||||
|
||||
def avg_cost_index = 0
|
||||
if (rMap.avg_cost) {
|
||||
// avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / 31.3526959
|
||||
avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / sd_avg_cost
|
||||
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
|
||||
}
|
||||
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
|
||||
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
|
||||
|
||||
|
||||
def year = CommonUtils.getBuildYear(d.finish_building_date)
|
||||
def sqmetre_rent = CommonDatautils.findNewSqmetreRent(d.city,d.region,d.name)
|
||||
def map = [:]
|
||||
map.id = d._id.toString()
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.name
|
||||
|
||||
map.avg_estate_charge = rMap.avg_estate_charge
|
||||
map.price_rank = rMap.price_rank
|
||||
map.avg_cost = rMap.avg_cost
|
||||
map.total_price = rMap.total_price
|
||||
|
||||
map.comp_value = potentiality
|
||||
map.avg_total_price = total_price_index
|
||||
map.month = month
|
||||
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
map.finish_date = year
|
||||
map.sell_avg_price = rMap.sell_avg_price
|
||||
if(sqmetre_rent){
|
||||
map.sqmetre_rent = sqmetre_rent
|
||||
}
|
||||
if(rMap.sell_avg_price && sqmetre_rent){
|
||||
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
|
||||
}
|
||||
if(d.household_count){
|
||||
map.household_count = d.household_count
|
||||
}
|
||||
if(d.lat){
|
||||
def lMap = [:]
|
||||
lMap.lon = d.lng/10000000000.0
|
||||
lMap.lat = d.lat/10000000000.0
|
||||
map.location = lMap
|
||||
}
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
|
||||
//修改多了会报错
|
||||
// def newUrl = targetUrl +"_update_by_query"
|
||||
// def must = []
|
||||
// def term1 = ["match_phrase":["id":d._id.toString()]]
|
||||
// must << term1
|
||||
// def source = "ctx._source.comp_value="+potentiality+";ctx._source.avg_total_price="+total_price_index+";ctx._source.month='2022-03'"+";ctx._source.m_date='"+DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")+"'"
|
||||
// def params = ["script":["source":source,"lang": "painless"], "query":["bool":["must":must]]]
|
||||
// httpSaveData(newUrl, params);
|
||||
|
||||
println totalCount.get() +" "+d.city+" "+d.region+" "+d.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await();
|
||||
}
|
||||
|
||||
def findAllDistrictDatas(def city){
|
||||
def detailMap = [:]
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
|
||||
for(def rs : houses){
|
||||
def id = rs.id
|
||||
def rMap = detailMap.get(id)
|
||||
if(!rMap){
|
||||
rMap = [:]
|
||||
detailMap.put(id, rMap)
|
||||
}
|
||||
rMap.put("total_price", rs.total_price)
|
||||
rMap.put("price_rank", rs.price_rank)
|
||||
rMap.put("avg_cost", rs.avg_cost)
|
||||
rMap.put("avg_estate_charge", rs.avg_estate_charge)
|
||||
rMap.put("sell_avg_price", rs.sell_avg_price)
|
||||
}
|
||||
return detailMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findDistrictDatasByCity(def city){
|
||||
def must = []
|
||||
def term1 = ["term":["city":city]]
|
||||
must << term1
|
||||
// def params = ["query":["bool":["must":must]]]
|
||||
def params = ["query":["match":["city":city]]]
|
||||
def s = httpSaveData(targetUrl+"/_search", params);
|
||||
def total = 0;
|
||||
if(s){
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
total = hits.total
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def calcuteAvgValues(detailMap){
|
||||
def prices = []
|
||||
def ranks = []
|
||||
def costs = []
|
||||
def charges = []
|
||||
detailMap.each{k,v->
|
||||
if(v.total_price){
|
||||
prices << v.total_price
|
||||
}
|
||||
if(v.price_rank){
|
||||
ranks << v.price_rank
|
||||
}
|
||||
if(v.avg_cost){
|
||||
costs << v.avg_cost
|
||||
}
|
||||
if(v.avg_estate_charge){
|
||||
charges << v.avg_estate_charge
|
||||
}
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
|
||||
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
||||
def avg_price = null
|
||||
def dx_avg_price = null
|
||||
def sd_avg_price = null
|
||||
if(prices){
|
||||
def v = prices.sum()/prices.size()
|
||||
avg_price = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
double pow_sum = 0
|
||||
prices.each {p->
|
||||
def sod = Math.abs(p - avg_price)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size())
|
||||
sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price))
|
||||
}
|
||||
def avg_rank = null
|
||||
def dx_avg_rank = null
|
||||
def sd_avg_rank = null
|
||||
if(ranks){
|
||||
def v = ranks.sum()/ranks.size()
|
||||
avg_rank = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
def pow_sum = 0;
|
||||
ranks.each {p->
|
||||
def sod = Math.abs(p - avg_rank)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size())
|
||||
sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank))
|
||||
}
|
||||
|
||||
def avg_cost = null
|
||||
def dx_avg_cost = null
|
||||
def sd_avg_cost = null
|
||||
if(costs){
|
||||
def v = costs.sum()/costs.size()
|
||||
avg_cost = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
double pow_sum = 0;
|
||||
costs.each {p->
|
||||
def sod = Math.abs(p - avg_cost)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size())
|
||||
sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost))
|
||||
}
|
||||
|
||||
|
||||
def avg_charge = null
|
||||
def dx_avg_charge = null
|
||||
def sd_avg_charge = null
|
||||
|
||||
if(charges){
|
||||
def v = charges.sum()/charges.size()
|
||||
avg_charge = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
def pow_sum = 0;
|
||||
charges.each {p->
|
||||
def sod = Math.abs(p - avg_charge)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size())
|
||||
sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge))
|
||||
}
|
||||
|
||||
def avgMap = [:]
|
||||
avgMap.put("avg_rank", avg_rank)
|
||||
avgMap.put("avg_price", avg_price)
|
||||
avgMap.put("avg_cost", avg_cost)
|
||||
avgMap.put("avg_charge", avg_charge)
|
||||
|
||||
avgMap.put("dx_avg_rank", dx_avg_rank)
|
||||
avgMap.put("dx_avg_price", dx_avg_price)
|
||||
avgMap.put("dx_avg_cost", dx_avg_cost)
|
||||
avgMap.put("dx_avg_charge", dx_avg_charge)
|
||||
|
||||
avgMap.put("sd_avg_rank", sd_avg_rank)
|
||||
avgMap.put("sd_avg_price", sd_avg_price)
|
||||
avgMap.put("sd_avg_cost", sd_avg_cost)
|
||||
avgMap.put("sd_avg_charge", sd_avg_charge)
|
||||
return avgMap;
|
||||
}
|
||||
|
||||
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1)
|
||||
public double Variance(def x) {
|
||||
int m=x.size();
|
||||
double sum=0;
|
||||
for(int i=0;i<m;i++){//求和
|
||||
sum+=x[i];
|
||||
}
|
||||
double dAve=sum/m;//求平均值
|
||||
double dVar=0;
|
||||
for(int i=0;i<m;i++){//求方差
|
||||
dVar+=(x[i]-dAve)*(x[i]-dAve);
|
||||
}
|
||||
return dVar/m;
|
||||
}
|
||||
|
||||
|
||||
def getAvgTotalPrice(city, startDate, endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1")
|
||||
@@ -309,7 +571,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
return datas
|
||||
}
|
||||
|
||||
def sortByDistance(d, list){
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
@@ -496,6 +758,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
|
||||
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
@@ -508,7 +771,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
@@ -547,7 +810,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null;
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class DsElasticUtils {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+210
-7
@@ -1,11 +1,20 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.codehaus.jackson.sym.NameN
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
class DistrictLocationUpdate {
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
@@ -19,20 +28,214 @@ class DistrictLocationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationUpdate()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.mapMatch()
|
||||
// task.updateLocationToDB()
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
|
||||
|
||||
|
||||
updateLocation(city, dists);
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"])
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"]
|
||||
for(def d : dists){
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(d.lng){
|
||||
lng = d.lng/10000000000.0
|
||||
lat = d.lat/10000000000.0
|
||||
}
|
||||
rets << [d.city, d.region, d.name, d.address, lng,lat]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
|
||||
if(!d['H'] && d['C']=="华侨城龙居御景二期"){
|
||||
index++
|
||||
// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德")
|
||||
// println d['A']+" "+d['B']+" "+d['C']
|
||||
// if(gdData){
|
||||
// d['G'] = gdData.district
|
||||
// d['H'] = gdData.name
|
||||
// d['I'] = gdData.address
|
||||
// d['J'] = gdData.type
|
||||
// d['K'] = gdData.lng
|
||||
// d['L'] = gdData.lat
|
||||
// d['M'] = gdData.score
|
||||
// if(d['E'] && d['K']){
|
||||
// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
|
||||
// d['N'] = distance
|
||||
// }
|
||||
// }
|
||||
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
}
|
||||
if(map){
|
||||
if(map.adname){
|
||||
d['G'] = map.adname
|
||||
}else if(map.region){
|
||||
d['G'] = map.region
|
||||
}
|
||||
d['H'] = map.name
|
||||
d['I'] = map.address
|
||||
if(map.type){
|
||||
d['J'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['J'] = map.estateType
|
||||
}
|
||||
d['K'] = map.lng
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.score
|
||||
if(d['E'] && d['K']){
|
||||
def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
|
||||
d['N'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateLocationToDB(){
|
||||
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
|
||||
if(d['N'] && d['N']>1000){
|
||||
if(d['M']>=0.4 && d['K']){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d['A'])
|
||||
query.put("region", d['B'])
|
||||
query.put("name", d['C'])
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
|
||||
Long lng0 = d['K'].toString().toDouble()*10000000000L
|
||||
Long lat0 = d['L'].toString().toDouble()*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble())
|
||||
Long lng2 = gmap.get("lng")*10000000000L
|
||||
Long lat2 = gmap.get("lat")*10000000000L
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("lng2", lng2)
|
||||
DbUtils.seaweed.update(query, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
|
||||
continue;
|
||||
}
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def queryPoiMap(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
|
||||
+2
-2
@@ -40,9 +40,9 @@ class LocationTest {
|
||||
// System.out.println(str);
|
||||
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.meituan
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class MatchTest {
|
||||
static main(args) {
|
||||
def task = new MatchTest()
|
||||
// task.run()
|
||||
// task.addNum()
|
||||
task.dongguang()
|
||||
// task.meituanDel()
|
||||
}
|
||||
|
||||
def meituanDel(){
|
||||
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
def flag = row['O']
|
||||
if(flag){
|
||||
def city = row['F']
|
||||
def region = row['G']
|
||||
def name = nameFileter(row['H'])
|
||||
def obj = null
|
||||
if(city == "东莞"){
|
||||
obj = CommonDatautils.findDistrict(city, null, name, "district")
|
||||
}else{
|
||||
obj = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
}
|
||||
if(obj){
|
||||
index++
|
||||
println index +" "+ city +' '+ region+" "+name
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(obj._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def dongguang(){
|
||||
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
|
||||
ExcelUtils.modify(path, { row ->
|
||||
def region = null
|
||||
if(row['B'] && row['B']=="NULL" ){
|
||||
region = null
|
||||
}
|
||||
def name = nameFileter(row['H'])
|
||||
def address = row['E']
|
||||
if (name && name.contains("湖畔新家")) {
|
||||
|
||||
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
|
||||
if(obj){
|
||||
def json = GaoDeApiUtils.geo(row['F'], address)
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
def twinshop = getStreetInfo(m.location)
|
||||
if(twinshop){
|
||||
def newRegion = RegionData.formatRegion("东莞",twinshop)
|
||||
if(newRegion){
|
||||
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
|
||||
if(dbObj){
|
||||
println "delete"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
|
||||
}else{
|
||||
println "update"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
// println address +" "+twinshop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def getStreetInfo(def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
}
|
||||
|
||||
def addNum(){
|
||||
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
index++
|
||||
row['A'] = index
|
||||
},0)
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def cityLevelMap = [:]
|
||||
def radioMap = [:]
|
||||
for(def city : citys){
|
||||
|
||||
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
|
||||
// radioMap.put(city, avg_radio)
|
||||
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
datas = calcuteLevel(datas);
|
||||
cityLevelMap.put(city, ["price":priceMap, "level":datas])
|
||||
println city +" 楼盘评级同步完成"
|
||||
|
||||
}
|
||||
|
||||
|
||||
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def region = null
|
||||
if(row['C'] && row['C']=="NULL" ){
|
||||
region = null
|
||||
}
|
||||
def name = row['D']
|
||||
def address = row['F']
|
||||
def city = RegionData.formatCity(row['B'])
|
||||
|
||||
if(name && !row['N']){
|
||||
if(city){
|
||||
def map = cityLevelMap.get(city)
|
||||
def rMap = null
|
||||
if(map){
|
||||
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
if(rMap.rank){
|
||||
row['Q'] = rMap.rank
|
||||
}
|
||||
if(rMap.level){
|
||||
row['P'] = rMap.level
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (name && row['N']) {
|
||||
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
|
||||
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
|
||||
// if (jsonObject != null && jsonObject.precise != -1) {
|
||||
// row['F'] = jsonObject.get("city")
|
||||
// row['G'] = jsonObject.get("district")
|
||||
// row['H'] = jsonObject.get("name")
|
||||
// row['I'] = jsonObject.get("propertyType")
|
||||
// row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
// }
|
||||
|
||||
if(row['G'] && row['H'] && row['I']){
|
||||
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
|
||||
// if(priceObj){
|
||||
// row['M'] = priceObj.avg_price
|
||||
// }
|
||||
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
|
||||
// if(sqmetreRentPrice){
|
||||
// row['N'] = sqmetreRentPrice
|
||||
// }
|
||||
|
||||
if(row['N'] && !row['O']){
|
||||
def avg_rent_radio = radioMap.get(city)
|
||||
row['O'] = (int)(row['N']/avg_rent_radio)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def getArroundDistrictPriceByAddress(def city, def address, def levels){
|
||||
def json = GaoDeApiUtils.geo(city, address)
|
||||
def location = null
|
||||
def detail = new BasicDBObject()
|
||||
def level = null
|
||||
def rank = null
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
location = m.location
|
||||
level = m.level
|
||||
}
|
||||
}
|
||||
if(location){
|
||||
def arrs = location.split(",")
|
||||
detail.lng = arrs[0].toString().toDouble()*10000000000L
|
||||
detail.lat = arrs[1].toString().toDouble()*10000000000L
|
||||
}
|
||||
if(detail.lng){
|
||||
detail.city = city
|
||||
|
||||
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
|
||||
if(list && list.size()>10){
|
||||
list = sortByDistance(detail, list);
|
||||
list = list[0..9]
|
||||
}
|
||||
def arroundAvgPrice = getArroundAvgPrice(list)
|
||||
if(arroundAvgPrice){
|
||||
|
||||
def mod = Integer.MAX_VALUE;
|
||||
for(def d : levels){
|
||||
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
|
||||
if(newMod<mod){
|
||||
mod = newMod
|
||||
rank = d.level
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def m = [:]
|
||||
m.level = level
|
||||
m.rank = rank
|
||||
return m
|
||||
}
|
||||
|
||||
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
}
|
||||
//排序
|
||||
Collections.sort(list, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
return list;
|
||||
}
|
||||
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
newLists << d;
|
||||
}
|
||||
}
|
||||
if(newLists){
|
||||
newLists.sort {a,b->
|
||||
def v1 = a.avg_price
|
||||
def v2 = b.avg_price
|
||||
return v1.compareTo(v2)
|
||||
}
|
||||
}
|
||||
return newLists;
|
||||
}
|
||||
|
||||
def calcuteLevel(def datas){
|
||||
def total = datas.size()
|
||||
def v = (int)(total/10)
|
||||
def start = 0;
|
||||
def end = 0;
|
||||
def map = [:]
|
||||
for(int i=1;i<=10;i++){
|
||||
end = end + v;
|
||||
if(i == 10){
|
||||
map.put(i, [start, total])
|
||||
}else{
|
||||
map.put(i, [start, end])
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for(def d=0;d<datas.size();d++){
|
||||
def dist = datas[d]
|
||||
|
||||
for (def o in map) {
|
||||
def ranges = o.value
|
||||
def m = ranges[0]
|
||||
def n = ranges[1]
|
||||
if(d==0){
|
||||
if(d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}else{
|
||||
if(d>m && d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getArroundAvgPrice(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
if(prices){
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
return avg_price;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
|
||||
}
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.gj.paimai.ChromeDriverPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
public class BeiKeCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
citys = citys.reverse()
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
// println i+" "+ m.city
|
||||
// if(m.city=="天津"){
|
||||
// start = i
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
getLogin(driver)
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
keys << ["city":d.city, "name":d.district_name]
|
||||
}
|
||||
for(def m : keys){
|
||||
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
|
||||
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
|
||||
if(list && list.size()==1){
|
||||
def dist = list[0]
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", m.city)
|
||||
query.put("region", null)
|
||||
query.put("district_name", m.name)
|
||||
// query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
for(def m : citys){
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def getPageUrls(driver, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findRegionByThirdPary(city, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", district_name)
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
|
||||
def dbo = null;
|
||||
for(def d : dbcursor){
|
||||
dbo = d
|
||||
}
|
||||
if(dbo){
|
||||
return dbo.region
|
||||
}
|
||||
return null;
|
||||
}
|
||||
def getContent(driver, detailUrl){
|
||||
driver.get(detailUrl)
|
||||
def str = driver.getPageSource()
|
||||
if(str && str.contains("<title>登录</title>")){
|
||||
getLogin(driver)
|
||||
return getContent(driver, detailUrl)
|
||||
}
|
||||
return str;
|
||||
}
|
||||
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
|
||||
if(detailUrls){
|
||||
for(def i=0;i<detailUrls.size();i++){
|
||||
def detailUrl = detailUrls[i]
|
||||
try{
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
|
||||
def content = getContent(driver, detailUrl)
|
||||
def doc = Jsoup.parse(content);
|
||||
def eles = JQueryUtils.select(".deal-bread a", doc)
|
||||
def detail = new BasicDBObject()
|
||||
if(eles && eles.size()>=4){
|
||||
detail.city = eles[0].text().replace("房产网","")
|
||||
detail.region = eles[2].text().replace("二手房成交","")
|
||||
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
|
||||
}
|
||||
if(detail.district_name) {
|
||||
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def getAllCityUrls(driver){
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||
def list = []
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("重庆")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+285
-124
@@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import javax.swing.JApplet
|
||||
import java.text.DateFormat
|
||||
import java.util.function.Consumer
|
||||
|
||||
@@ -41,45 +42,138 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
|
||||
def doc = getDocument();
|
||||
def detailUrls = getDetailLinks(doc);
|
||||
def rets = batchParse(detailUrls);
|
||||
saveToDB(rets);
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def proviceMap = getDocument(driver);
|
||||
def citys = getCity(driver,proviceMap)
|
||||
def detailUrls = getDetailLinks(driver, citys);
|
||||
|
||||
|
||||
def getDetailLinks(doc){
|
||||
def urls = []
|
||||
|
||||
def getDetailLinks(driver, cityList){
|
||||
|
||||
def zxcitys = ["天津市","北京市","上海市","重庆市"]
|
||||
for(def cityMap : cityList){
|
||||
def citys = cityMap.citys
|
||||
|
||||
citys.each{city,id->
|
||||
def urls = []
|
||||
if(!zxcitys.contains(cityMap.provice)){
|
||||
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
|
||||
getDocument(url, driver)
|
||||
}else{
|
||||
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
|
||||
getDocument(url, driver)
|
||||
}
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def count = JQueryUtils.selectText(".count", doc)
|
||||
def totalPage = getTotalPage(count, 12)
|
||||
if(totalPage){
|
||||
for(def i=1;i<=totalPage;i++){
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
|
||||
if(eles){
|
||||
for(def el :eles){
|
||||
def href = el.attr("href")
|
||||
def map = [:]
|
||||
map.province = cityMap.provice
|
||||
map.city = city
|
||||
map.url = "https://paimai.caa123.org.cn"+href
|
||||
urls << map
|
||||
}
|
||||
}
|
||||
try{
|
||||
driver.findElement(By.className("next")).click()
|
||||
}catch(Exception e){
|
||||
Thread.sleep(8000)
|
||||
driver.findElement(By.className("next")).click()
|
||||
}
|
||||
}
|
||||
|
||||
def rets = batchParse(urls, driver);
|
||||
saveToDB(rets);
|
||||
println city +" "+urls.size()
|
||||
}
|
||||
}
|
||||
println cityMap.provice +" 完成"
|
||||
|
||||
}
|
||||
|
||||
//return urls;
|
||||
}
|
||||
|
||||
def getTotalPage(def total, def pageSize){
|
||||
if(total){
|
||||
def mode = total.toInteger()/pageSize
|
||||
if(mode > mode.toInteger()){
|
||||
mode = mode.toInteger() +1
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
def getCity(driver, proviceMap){
|
||||
def rows = []
|
||||
def zxcitys = ["天津市","北京市","上海市","重庆市"]
|
||||
proviceMap.each {k,v->
|
||||
if(!zxcitys.contains(k)){
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc)
|
||||
def cityMap = [:]
|
||||
for(def c : cityEles){
|
||||
def cityid = c.attr("data-cityid")
|
||||
def city = c.text()
|
||||
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
|
||||
cityMap.put(city, cityid)
|
||||
}
|
||||
|
||||
}
|
||||
def map = [:]
|
||||
map.provice = k
|
||||
map.provice_id = v
|
||||
map.citys = cityMap
|
||||
rows << map
|
||||
}else{
|
||||
def map = [:]
|
||||
map.provice = k
|
||||
map.provice_id = v
|
||||
def cityMap = [:]
|
||||
cityMap.put(k, v)
|
||||
map.citys = cityMap
|
||||
rows << map
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
def getDocument(driver){
|
||||
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def provinceMap = [:]
|
||||
if(doc){
|
||||
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
|
||||
if(eles){
|
||||
for(def el :eles){
|
||||
def href = el.attr("href")
|
||||
urls << "https://paimai.caa123.org.cn"+href
|
||||
def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc)
|
||||
for(def ele : proviceEles){
|
||||
def provice = ele.text()
|
||||
def value = ele.attr("data-proviceid")
|
||||
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
|
||||
provinceMap.put(provice, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
return provinceMap;
|
||||
}
|
||||
|
||||
def getDocument(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
||||
def batchParse(def urls, driver){
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
def batchParse(def urls){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def ret = []
|
||||
for(def url :urls){
|
||||
driver.get(url);
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def res = parse(url, doc);
|
||||
for(def map :urls){
|
||||
def doc = getDocument(map.url, driver)
|
||||
def res = parse(map.province, map.city, map.url, doc);
|
||||
if(res){
|
||||
ret << res
|
||||
}
|
||||
@@ -87,111 +181,167 @@ def batchParse(def urls){
|
||||
return ret;
|
||||
}
|
||||
|
||||
def parse(url,doc){
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parse(province, city, url,doc){
|
||||
if(doc){
|
||||
def result = [:]
|
||||
// def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
// result.address = address
|
||||
if(result.address){
|
||||
def rMap = OpenApiUtils.matchDistrict(result.address)
|
||||
if(rMap) {
|
||||
result.matchCity = rMap.matchCity
|
||||
result.matchRegion = rMap.matchRegion
|
||||
result.matchName= rMap.matchName
|
||||
result.matchAddress = rMap.matchAddress
|
||||
result.province = province
|
||||
result.city = city
|
||||
def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
result.address = address
|
||||
if(result.address && !result.address.contains("测试")) {
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
||||
if(finishDateStr) {
|
||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss");
|
||||
}
|
||||
result.status = JQueryUtils.selectText([".bd_state"], doc)
|
||||
result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc)
|
||||
result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc)
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",","")
|
||||
result.tradePrice = curPriceStr.toInteger()
|
||||
}
|
||||
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
||||
if (finishDateStr) {
|
||||
try{
|
||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
|
||||
if(evaluationPriceStr) {
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "")
|
||||
if(evaluationPriceStr.isInteger()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
|
||||
if(depositPriceStr) {
|
||||
depositPriceStr = depositPriceStr.replace(",", "")
|
||||
if(depositPriceStr.isInteger()) {
|
||||
result.depositPrice = depositPriceStr.toInteger()
|
||||
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
||||
if (curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "")
|
||||
result.tradePrice = curPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
|
||||
if(addPriceStr) {
|
||||
addPriceStr = addPriceStr.replace(",", "")
|
||||
if(addPriceStr.isInteger()) {
|
||||
result.addPrice = addPriceStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
|
||||
if(startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc)
|
||||
result.retain = JQueryUtils.selectText(["#retain"], doc)
|
||||
|
||||
def imgs =JQueryUtils.select(".sf-pic-slide img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if(href.contains("http")) {
|
||||
imgLinks << href
|
||||
}else {
|
||||
imgLinks << url.substring(0, url.indexOf(":")+1)+href
|
||||
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
|
||||
if (evaluationPriceStr) {
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "")
|
||||
if (evaluationPriceStr.isNumber()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
def desc = JQueryUtils.selectText(["#J_desc"], doc)
|
||||
if(!desc){
|
||||
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
}
|
||||
if(desc) {
|
||||
if(desc.contains("别墅")) {
|
||||
result.address = desc.substring(0, desc.indexOf("别墅")+2)
|
||||
}else if(desc.contains("号楼")) {
|
||||
result.address = desc.substring(0, desc.indexOf("号楼")+2)
|
||||
}else if(desc.contains("房屋")) {
|
||||
result.address = desc.substring(0, desc.indexOf("房屋")+2)
|
||||
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
|
||||
if (depositPriceStr) {
|
||||
depositPriceStr = depositPriceStr.replace(",", "")
|
||||
if (depositPriceStr.isNumber()) {
|
||||
result.depositPrice = depositPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def descr = JQueryUtils.selectText(["body"], doc)
|
||||
if(descr) {
|
||||
def area = getAreaNumber(descr, 5000);
|
||||
if(area){
|
||||
result.area = area.toDouble()
|
||||
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
|
||||
if (addPriceStr) {
|
||||
addPriceStr = addPriceStr.replace(",", "")
|
||||
if (addPriceStr.isNumber()) {
|
||||
result.addPrice = addPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
if(result.area && result.tradePrice){
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
||||
result.url = url
|
||||
return result;
|
||||
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
|
||||
if (startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
|
||||
if (startPriceStr.isNumber()) {
|
||||
result.startPrice = startPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc)
|
||||
result.retain = JQueryUtils.selectText(["#retain"], doc)
|
||||
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
|
||||
|
||||
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
|
||||
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
|
||||
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
|
||||
}
|
||||
|
||||
try{
|
||||
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
|
||||
if (imgs) {
|
||||
def imgLinks = []
|
||||
for (def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if (href.contains("http")) {
|
||||
imgLinks << href
|
||||
} else {
|
||||
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
def desc = JQueryUtils.selectText(["#J_desc"], doc)
|
||||
if (!desc) {
|
||||
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
}
|
||||
if (desc) {
|
||||
if (desc.contains("别墅")) {
|
||||
result.address = desc.substring(0, desc.indexOf("别墅") + 2)
|
||||
} else if (desc.contains("号楼")) {
|
||||
result.address = desc.substring(0, desc.indexOf("号楼") + 2)
|
||||
} else if (desc.contains("房屋")) {
|
||||
result.address = desc.substring(0, desc.indexOf("房屋") + 2)
|
||||
}
|
||||
}
|
||||
|
||||
def descr = JQueryUtils.selectText(["body"], doc)
|
||||
if (descr) {
|
||||
def area = getAreaNumber(descr, 5000);
|
||||
if (area) {
|
||||
result.area = area.toDouble()
|
||||
}
|
||||
}
|
||||
if (result.area && result.tradePrice) {
|
||||
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
|
||||
}
|
||||
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
if(result.startDate){
|
||||
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
||||
result.url = url
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -207,11 +357,13 @@ def saveToDB(rets){
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
obj.source= "中拍网"
|
||||
if(dbo){
|
||||
obj.remove("url")
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.save(obj)
|
||||
}
|
||||
}
|
||||
@@ -229,16 +381,25 @@ def getAreaNumber(input, maxArea) {
|
||||
def area = null;
|
||||
|
||||
for (def val : list) {
|
||||
if(val.toString().contains(" ")){
|
||||
val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size())
|
||||
}
|
||||
if(val.contains("室")){
|
||||
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
|
||||
}
|
||||
def v = val.replace("ƽ·½Ã×", "").replace("©O", "").replace("ƽ", "").replace("·½Ã×", "").replace(",", "").replace("m2", "");
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
if(v && v.toString().isNumber()){
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (area) {
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.apache.commons.codec.binary.Base64
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import sun.font.Font2D
|
||||
import sun.font.Font2DHandle
|
||||
import sun.font.TrueTypeFont
|
||||
import sun.font.TrueTypeGlyphMapper
|
||||
|
||||
import java.awt.Font
|
||||
import java.awt.font.FontRenderContext
|
||||
import java.awt.font.GlyphVector
|
||||
import java.awt.geom.AffineTransform
|
||||
import java.lang.reflect.Field
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataUtil {
|
||||
|
||||
|
||||
static def checkMinDate(curDate, checkDate){
|
||||
if(curDate){
|
||||
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
|
||||
if(curDate.getTime()<minDate.getTime()){
|
||||
return null
|
||||
}
|
||||
}
|
||||
return curDate
|
||||
}
|
||||
|
||||
static def checkMaxDate(curDate, checkDate){
|
||||
if(curDate){
|
||||
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
|
||||
if(curDate.getTime()>maxDate.getTime()){
|
||||
return null
|
||||
}
|
||||
}
|
||||
return curDate
|
||||
}
|
||||
static def getAreaNumber(def input, def maxArea) {
|
||||
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
|
||||
Matcher matcher = pattern.matcher(input)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() > 0) {
|
||||
def area = null
|
||||
|
||||
for (def val : list) {
|
||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
|
||||
if (maxArea) {
|
||||
if (v.isDouble()) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (area) {
|
||||
return area
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def getPaiMaiType(def s) {
|
||||
if (s) {
|
||||
if (s.toString().contains("结束")) {
|
||||
return "已完成"
|
||||
} else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
|
||||
return "进行中"
|
||||
} else if (s.toString().contains("即将开始")) {
|
||||
return "即将开始"
|
||||
} else if (s.toString().contains("流拍")) {
|
||||
return "流拍"
|
||||
} else if (s.toString().contains("撤回")) {
|
||||
return "撤回"
|
||||
} else if (s.toString().contains("暂缓")) {
|
||||
return "暂缓"
|
||||
} else if (s.toString().contains("中止")) {
|
||||
return "中止"
|
||||
} else if (s.toString().contains("失败")) {
|
||||
return "失败"
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
static def getDateRange(def s) {
|
||||
s = s.replace("起", "")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
def str = matcher.group()
|
||||
list.add(str)
|
||||
// if(str) {
|
||||
// def indx1 = str.indexOf("年")
|
||||
// def indx2 = str.indexOf("月")
|
||||
// def month = str.substring(indx1+1, indx2);
|
||||
// if(month.toInteger()<12) {
|
||||
// list.add(str);
|
||||
// }
|
||||
// }
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "").replace(" ", "")
|
||||
def array = s1.split("至")
|
||||
return array
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def getStartPrice(def s) {
|
||||
Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1.contains("万元")) {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
|
||||
return (int) (s1.toDouble() * 10000)
|
||||
} else {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
|
||||
if (s1.isInteger()) {
|
||||
return s1.toInteger()
|
||||
} else {
|
||||
return (int) (s1.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def getEstateType(def s) {
|
||||
Pattern pattern = Pattern.compile('房屋类型:.*;')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1) {
|
||||
s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
|
||||
return s1.replace(")", "").replace("】", "")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def findEstateType(def s) {
|
||||
if (s) {
|
||||
def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
|
||||
for (def type : villTypes) {
|
||||
if (s.contains(type)) {
|
||||
return "别墅"
|
||||
}
|
||||
}
|
||||
|
||||
def districtTypes = ["住宅", "公寓"]
|
||||
for (def type : districtTypes) {
|
||||
if (s.contains(type)) {
|
||||
return "普通住宅"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static def filterChar(def detailMap) {
|
||||
def newDetailMap = [:]
|
||||
detailMap.each { def k, v ->
|
||||
if (k.toString().contains(".")) {
|
||||
newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
|
||||
} else {
|
||||
newDetailMap.put(k, v)
|
||||
}
|
||||
}
|
||||
return newDetailMap
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* font-secret字符串专用解密工具
|
||||
*
|
||||
* @param key 密匙
|
||||
* @param encodeString 加密后的字符串
|
||||
* @return 解密后的字符串
|
||||
*/
|
||||
static String decodeString(String key, String encodeString) {
|
||||
if (!key) {
|
||||
return ""
|
||||
}
|
||||
try {
|
||||
//base64解码,初始化字体
|
||||
Base64 base64 = new Base64()
|
||||
byte[] ss = base64.decodeBase64(key)
|
||||
InputStream inputStream = new ByteArrayInputStream(ss)
|
||||
Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
|
||||
FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
|
||||
GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
|
||||
|
||||
//获取font中字形的映射关系,字段为private,使用反射
|
||||
Class<?> clazz = Font.class
|
||||
Field[] fs = clazz.getDeclaredFields()
|
||||
Font2DHandle font2DHandle = null
|
||||
for (int i = 0; i < fs.length; i++) {
|
||||
fs[i].setAccessible(true)// 将目标属性设置为可以访问
|
||||
if (fs[i].getName().equals("font2DHandle")) {
|
||||
font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//得到映射关系
|
||||
Font2D font2D = font2DHandle.font2D
|
||||
TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
|
||||
TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
|
||||
|
||||
//开始解密,encodeString为加密后的字符串
|
||||
StringBuffer buffer = new StringBuffer()
|
||||
char[] chars = encodeString.toCharArray()
|
||||
for (int i = 0; i < chars.length; i++) {
|
||||
buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
|
||||
}
|
||||
return buffer.toString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
|
||||
|
||||
static String urlToSiteName(String url) {
|
||||
if (StringUtils.isEmpty(url)) {
|
||||
return null
|
||||
}
|
||||
|
||||
for (String suffix : SITE_SUFFIXS) {
|
||||
if (url.indexOf(suffix) > -1) {
|
||||
String s = StringUtils.substringBefore(url, suffix)
|
||||
if (s.indexOf(".") > -1) {
|
||||
s = StringUtils.substringAfterLast(s, ".")
|
||||
if (StringUtils.isNotBlank(s)) {
|
||||
return s + suffix
|
||||
}
|
||||
} else if (s.indexOf("://") > -1) {
|
||||
s = StringUtils.substringAfterLast(s, "://")
|
||||
if (StringUtils.isNotBlank(s)) {
|
||||
return s + suffix
|
||||
}
|
||||
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
|
||||
return s + suffix
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static double getRound(int scale, double dSource) {
|
||||
BigDecimal deSource = new BigDecimal(dSource)
|
||||
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
|
||||
}
|
||||
|
||||
static def filterEmpty(def s){
|
||||
if(s && s.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
}
|
||||
+2
-1
@@ -514,8 +514,9 @@ class GaoDeUtils {
|
||||
// "2a1fb75990f2b39f6221e50892dd3988"
|
||||
// ]
|
||||
|
||||
//35925efd7333f1df52145e51b8e4e56a 15021384995
|
||||
def KeyList = [
|
||||
"10bcb421d2ee4e51ed9268d6dca58ad1"
|
||||
"10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975
|
||||
]
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
|
||||
+348
@@ -0,0 +1,348 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import javafx.scene.input.DataFormat
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class IcbcCrawl {
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new IcbcCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get(url)
|
||||
driver.findElementById("category_SM310000000000000006").click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
|
||||
def urls = getAllDetailLinks(driver)
|
||||
println "共:"+urls.size()
|
||||
|
||||
detailCrawling(urls, driver)
|
||||
|
||||
}
|
||||
|
||||
def getAllDetailLinks(driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
while (!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
|
||||
if(eles){
|
||||
for(def e : eles){
|
||||
def href = e.select(".li_2_name a").attr("href")
|
||||
urls << "https://gf.trade.icbc.com.cn"+href
|
||||
}
|
||||
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(nextPage){
|
||||
driver.findElement(By.linkText("下一页")).click();
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
// if(urls.size()>=10){
|
||||
// stop = true
|
||||
// }
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def detailCrawling(def urls, driver){
|
||||
def rets = []
|
||||
for(def url : urls){
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def result = parser(doc, url)
|
||||
if (result) {
|
||||
rets << result
|
||||
println rets.size()
|
||||
}
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
saveData(rets)
|
||||
print "完"
|
||||
}
|
||||
}
|
||||
|
||||
def parser(doc, url){
|
||||
def result = [:]
|
||||
if(doc){
|
||||
def title = JQueryUtils.selectText("#prodnameb", doc)
|
||||
def id = url.substring(url.indexOf("="), url.length())
|
||||
if(title){
|
||||
|
||||
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
|
||||
if(cityStr && cityStr.contains("...")){
|
||||
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
|
||||
if(cityEles){
|
||||
cityStr = cityEles[0].attr("title")
|
||||
}
|
||||
}
|
||||
if(cityStr && cityStr.contains("省")){
|
||||
result.province = cityStr.substring(0, cityStr.indexOf("省"))
|
||||
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
|
||||
}else{
|
||||
if(cityStr.contains("自治区")){
|
||||
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
|
||||
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
|
||||
}else{
|
||||
result.city = cityStr.substring(0, cityStr.indexOf("市"))
|
||||
}
|
||||
}
|
||||
|
||||
if(!result.city){
|
||||
return null
|
||||
}
|
||||
result.title = title
|
||||
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
|
||||
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
|
||||
|
||||
def t1 = JQueryUtils.selectText("#pricep", doc)
|
||||
if(t1 && t1.contains("原因")){
|
||||
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
|
||||
}
|
||||
result.type = JQueryUtils.selectText("dl:contains(类 型:) dd", doc)
|
||||
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
|
||||
if(enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
|
||||
if(callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
|
||||
if(curPriceStr.isInteger()) {
|
||||
result.curPrice = curPriceStr.toInteger()
|
||||
}else {
|
||||
result.curPrice = (int)(curPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
|
||||
if(startPriceEles) {
|
||||
def startPriceStr = startPriceEles[0].attr("title")
|
||||
if(startPriceStr){
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
|
||||
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(startPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
|
||||
if(addPriceEles){
|
||||
def addPriceStr = addPriceEles[0].attr("title")
|
||||
if(addPriceStr){
|
||||
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
|
||||
if(addPriceStr) {
|
||||
result.addPrice = addPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
|
||||
if(evaluationPriceEles){
|
||||
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
|
||||
if(evaluationPriceStr){
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
|
||||
if(evaluationPriceStr.isInteger()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
|
||||
}else {
|
||||
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
|
||||
if(depositPriceEles){
|
||||
def v = depositPriceEles[0].attr("title")
|
||||
if(v){
|
||||
v = v.replace(",", "").replace("¥","")
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
|
||||
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
|
||||
result.court = JQueryUtils.selectText("#goToCourt", doc)
|
||||
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
|
||||
if(mobileEles){
|
||||
result.mobile = mobileEles[0].attr("title")
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = PaimaiDataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
|
||||
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
|
||||
}
|
||||
}
|
||||
|
||||
def trs = JQueryUtils.select([".rygtable table tr"], doc)
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
if(d_tds[2].text() && d_tds[2].text()!="/"){
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}
|
||||
|
||||
}else {
|
||||
if(d_tds[1].text() && d_tds[1].text()!="/") {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def imgs =JQueryUtils.select("#scroll li img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if(href.contains("icbc.com")) {
|
||||
imgLinks << "https:"+href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
|
||||
def detailStr = JQueryUtils.selectText(".detail_list", doc)
|
||||
if(detailStr){
|
||||
def area = DataUtil.getAreaNumber(detailStr, 5000)
|
||||
if(area) {
|
||||
result.area = area.toDouble()
|
||||
if(result.tradePrice) {
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = detailStr.toString()
|
||||
result.url = url
|
||||
result.mDate = new Date()
|
||||
result.source = "融e够"
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getCallCount(id){
|
||||
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
|
||||
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
println jsonObject
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
}
|
||||
+533
@@ -0,0 +1,533 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import javax.sound.midi.MidiSystem
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class JdPaimaiCrawl {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new JdPaimaiCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def existsIds = findAllIds()
|
||||
println "已存在:"+existsIds.size()
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def links = getList(driver)
|
||||
def ids = getDetailLinks(links, driver)
|
||||
println "共:"+ids.size()
|
||||
parserList( existsIds,ids, driver)
|
||||
|
||||
}
|
||||
|
||||
def getList(driver){
|
||||
def list = []
|
||||
for(def i=1;i<=32;i++){
|
||||
if(i==8){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
||||
}else if(i==5){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
||||
}else if(i==17){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
||||
}else if(i==19){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
||||
}else{
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
||||
}
|
||||
}
|
||||
def links = []
|
||||
for(def url : list){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
//def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
|
||||
if(doc){
|
||||
def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
|
||||
JSONObject obj = JSONObject.fromObject(json);
|
||||
if(obj && obj.total){
|
||||
def total = obj.total
|
||||
def page = total/40
|
||||
def p = page.toInteger()
|
||||
if(page > p){
|
||||
p = p +1
|
||||
}
|
||||
|
||||
for(def i=1;i<=p;i++){
|
||||
links << url+"&page="+i
|
||||
}
|
||||
// if(links){
|
||||
// break;
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
def getDetailLinks(links, driver){
|
||||
def rets = []
|
||||
def citys = ["北京","上海","天津","重庆"]
|
||||
for(def link : links){
|
||||
driver.get(link)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc) {
|
||||
def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
|
||||
JSONObject obj = JSONObject.fromObject(json);
|
||||
if (obj && obj.ls) {
|
||||
for(def o : obj.ls){
|
||||
def map = [:]
|
||||
if(citys.contains(o.province)){
|
||||
map.province = o.province
|
||||
map.city = o.province
|
||||
map.region = o.city
|
||||
}else{
|
||||
map.province = o.province
|
||||
map.city = o.city
|
||||
}
|
||||
map.id = o.id.toString()
|
||||
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def parserList(existsIds,list, driver){
|
||||
def rets = []
|
||||
for(def map : list) {
|
||||
if(!existsIds.contains(map.id)){
|
||||
def url = "https://paimai.jd.com/" + map.id
|
||||
def doc = getDocument(url, driver)
|
||||
if (doc) {
|
||||
def result = parsering(doc, url, map.province, map.city, map.region)
|
||||
if (result) {
|
||||
rets << result
|
||||
}
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
saveData(rets)
|
||||
print "完"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
|
||||
def parsering(doc, def url, def province, def city, def region){
|
||||
def result = [:]
|
||||
if(doc) {
|
||||
result.proince = province
|
||||
result.city = city
|
||||
if(region){
|
||||
result.region = region
|
||||
}
|
||||
def title = JQueryUtils.selectText([".pm-name"], doc)
|
||||
result.title = title
|
||||
result.address = JQueryUtils.selectText([".pm-location em"], doc)
|
||||
if(result.address) {
|
||||
//调用获取城市
|
||||
// if(!city) {
|
||||
// city = RegionData.formatCity(cityMap.name)
|
||||
// }
|
||||
|
||||
if(city) {
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
|
||||
if(enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
|
||||
if(callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "")
|
||||
if(curPriceStr.isInteger()) {
|
||||
result.curPrice = curPriceStr.toInteger()
|
||||
}else {
|
||||
result.curPrice = (int)(curPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
|
||||
if(startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "")
|
||||
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(startPriceStr.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
|
||||
if(tradePriceStr) {
|
||||
tradePriceStr = tradePriceStr.replace(",", "")
|
||||
if(tradePriceStr.isInteger()) {
|
||||
result.tradePrice = tradePriceStr.toInteger()
|
||||
}else {
|
||||
result.tradePrice = (int)(tradePriceStr.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
|
||||
if(finishDateStr && finishDateStr.contains("结束时间")) {
|
||||
finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
|
||||
}
|
||||
|
||||
def lis = JQueryUtils.select(".description ul li", doc)
|
||||
if(lis && lis.size()) {
|
||||
for(def li : lis) {
|
||||
def str = li.select("span").text()
|
||||
if(str.contains("起拍")) {
|
||||
def v= li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.startPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(v.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}else if(str.contains("评估价")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.evaluationPrice = (int)(v.toDouble())
|
||||
}
|
||||
}else if(str.contains("加价幅度")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
result.addPrice = v.toDouble().toInteger()
|
||||
}
|
||||
}else if(str.contains("保证金")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}else if(str.contains("延时周期")) {
|
||||
|
||||
result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
|
||||
}else if(str.contains("优先竞买权人")) {
|
||||
result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}else {
|
||||
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
if(detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("href")
|
||||
if(href.contains("http")) {
|
||||
imgLinks << href
|
||||
}else {
|
||||
imgLinks << url.substring(0, url.indexOf(":")+1)+href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//解析面积
|
||||
def article = JQueryUtils.selectText([".floor-content"], doc)
|
||||
if(article) {
|
||||
def area = DataUtil.getAreaNumber(article, 5000)
|
||||
if(!area) {
|
||||
article = JQueryUtils.selectText(["body"], doc)
|
||||
if(article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
|
||||
if(area) {
|
||||
result.area = area.toDouble()
|
||||
if(result.tradePrice) {
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = DataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
|
||||
for(def j=0;j<dateRangs.size();j++) {
|
||||
def date = dateRangs[j]
|
||||
def pattern = null
|
||||
if(date.endsWith("日时")||date.endsWith("日")) {
|
||||
pattern = "yyyy年MM月dd日"
|
||||
}else {
|
||||
|
||||
def ind = date.indexOf("日")
|
||||
if(date.indexOf("时")==-1 && date.toString().length()>ind+1) {
|
||||
date = date +"时"
|
||||
}
|
||||
|
||||
pattern = "yyyy年MM月dd日HH时"
|
||||
}
|
||||
if(j==0) {
|
||||
try{
|
||||
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
}else {
|
||||
if(!result.finishDate) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!result.startPrice) {
|
||||
result.startPrice = DataUtil.getStartPrice(s1)
|
||||
}
|
||||
|
||||
def estateType = DataUtil.findEstateType(s1)
|
||||
if(estateType) {
|
||||
result.matchEstateType = estateType
|
||||
}
|
||||
|
||||
}
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
if(result.startDate){
|
||||
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
|
||||
result.url = url
|
||||
result.city = city
|
||||
result.mDate = new Date()
|
||||
result.source = "京东"
|
||||
return result;
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
def findAllIds(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", Pattern.compile(".*jd.com.*"))
|
||||
def dbcursor = paimaiResultColl.find(q)
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
def url = d.url
|
||||
if(url){
|
||||
def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
}
|
||||
+45
-12
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
@@ -34,8 +35,9 @@ public class LianjiaCrawlByChrome {
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
task.standardRegion()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -94,22 +96,44 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
// System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
// def cityMap = getAllCityUrls(driver)
|
||||
// getLogin(driver)
|
||||
// crawlData(driver, cityMap)
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
if(d.region){
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if(newRegion && newRegion!=d.region){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d.city)
|
||||
query.put("region", d.region)
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", newRegion)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["Á´¼ÒÔÚÏß","±´¿ÇÕÒ·¿"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
@@ -131,6 +155,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
println m.city+ " "+m.name +" "+dist.region
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
@@ -162,7 +187,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
@@ -193,7 +218,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -202,6 +227,10 @@ public class LianjiaCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -219,6 +248,9 @@ public class LianjiaCrawlByChrome {
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
@@ -388,10 +420,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["³¤´º","¼ªÁÖ"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if(!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+53
-26
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
@@ -94,32 +95,32 @@ class PaimaiDataUtil {
|
||||
}
|
||||
|
||||
|
||||
public static def getDateRange(def s) {
|
||||
s = s.replace("起","")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
def str = matcher.group()
|
||||
list.add(str);
|
||||
// if(str) {
|
||||
// def indx1 = str.indexOf("年")
|
||||
// def indx2 = str.indexOf("月")
|
||||
// def month = str.substring(indx1+1, indx2);
|
||||
// if(month.toInteger()<12) {
|
||||
// list.add(str);
|
||||
// }
|
||||
// }
|
||||
}
|
||||
if(list.size()>=1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "").replace(" ","")
|
||||
def array = s1.split("至");
|
||||
return array
|
||||
}
|
||||
return null;
|
||||
}
|
||||
// public static def getDateRange(def s) {
|
||||
// s = s.replace("起","")
|
||||
// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
|
||||
// Matcher matcher = pattern.matcher(s);
|
||||
//
|
||||
// List<String> list = new ArrayList();
|
||||
// while (matcher.find()) {
|
||||
// def str = matcher.group()
|
||||
// list.add(str);
|
||||
//// if(str) {
|
||||
//// def indx1 = str.indexOf("年")
|
||||
//// def indx2 = str.indexOf("月")
|
||||
//// def month = str.substring(indx1+1, indx2);
|
||||
//// if(month.toInteger()<12) {
|
||||
//// list.add(str);
|
||||
//// }
|
||||
//// }
|
||||
// }
|
||||
// if(list.size()>=1) {
|
||||
// String s1 = list.get(0)
|
||||
// s1 = s1.replace("法院将于", "").replace(" ","")
|
||||
// def array = s1.split("至");
|
||||
// return array
|
||||
// }
|
||||
// return null;
|
||||
// }
|
||||
|
||||
|
||||
public static def getStartPrice(def s) {
|
||||
@@ -159,4 +160,30 @@ class PaimaiDataUtil {
|
||||
return newDetailMap;
|
||||
}
|
||||
|
||||
public static def getDateRange(def s) {
|
||||
s = s.replace("起","")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)');
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>=1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "")
|
||||
def array = s1.split("至");
|
||||
return array
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static Date parseFromTightByPattern(String s, String pattern){
|
||||
if (s == null || s.length() == 0){
|
||||
return null
|
||||
}
|
||||
SimpleDateFormat format = new SimpleDateFormat(pattern);
|
||||
return format.parse(s)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+242
-24
@@ -7,34 +7,174 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
import java.util.zip.GZIPOutputStream
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import java.text.SimpleDateFormat;
|
||||
|
||||
class PaimaiTest {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiTest();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134"
|
||||
def con = find(url);
|
||||
parser(con, url);
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def url = "https://s.gpai.net/sf/search.do?at=376"
|
||||
def provinces = getProvice(url, driver)
|
||||
def cityUrls = getCity(provinces, driver)
|
||||
getDetailUrls(cityUrls, driver)
|
||||
|
||||
|
||||
}
|
||||
def getDetailUrls(cityUrls, driver){
|
||||
for(def cMap : cityUrls){
|
||||
driver.get("https:"+cMap.url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc)
|
||||
|
||||
def parser(s, url){
|
||||
Document doc = Jsoup.parse(s);
|
||||
if(total){
|
||||
def totalPage = getTotalPage(total.toInteger(), 20)
|
||||
|
||||
def detailDatas = []
|
||||
for(def i=1;i<=totalPage;i++){
|
||||
driver.get("https:"+cMap.url+"&page="+i)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc)
|
||||
for(def ele : detailEles){
|
||||
def dMap = [:]
|
||||
dMap.proince = cMap.proince
|
||||
dMap.city = cMap.city
|
||||
dMap.url = ele.select("a").attr("href")
|
||||
detailDatas << dMap
|
||||
}
|
||||
}
|
||||
|
||||
def results = parser(detailDatas, driver, cMap.proince, cMap.city)
|
||||
saveData(results)
|
||||
println cMap.proince+" "+cMap.city+" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def getTotalPage(def total, def pageSize){
|
||||
if(total){
|
||||
def mode = total.toInteger()/pageSize
|
||||
if(mode > mode.toInteger()){
|
||||
mode = mode.toInteger() +1
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
def getCity(datas, driver){
|
||||
def citys = ["上海","北京","天津","重庆"]
|
||||
def cityList = []
|
||||
for(def data : datas){
|
||||
driver.get("https:"+data.href)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
if(citys.contains(data.proince)){
|
||||
def links = JQueryUtils.select(".condition .s-subtab a", doc)
|
||||
for(def ele : links) {
|
||||
def cityMap = [:]
|
||||
cityMap.proince = data.proince
|
||||
cityMap.city = data.proince
|
||||
cityMap.region =ele.text()
|
||||
cityMap.url = data.href
|
||||
cityList << cityMap
|
||||
}
|
||||
}else{
|
||||
def links = JQueryUtils.select(".condition .s-subtab a", doc)
|
||||
for(def ele : links){
|
||||
if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
|
||||
def cityMap = [:]
|
||||
cityMap.proince = data.proince
|
||||
cityMap.city = ele.text()
|
||||
cityMap.url = ele.attr("href")
|
||||
cityList << cityMap
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityList
|
||||
}
|
||||
|
||||
def getProvice(url, driver){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def datas = []
|
||||
def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"]
|
||||
if(doc){
|
||||
def eles = JQueryUtils.select(".condition a", doc)
|
||||
for(def e : eles){
|
||||
def data = [:]
|
||||
if(!keys.contains(e.text())){
|
||||
data.proince = e.text()
|
||||
data.href = e.attr("href")
|
||||
datas << data
|
||||
}
|
||||
}
|
||||
}
|
||||
return datas
|
||||
}
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parser(detailDatas, driver, proince, city){
|
||||
def results = []
|
||||
for(def m : detailDatas){
|
||||
// try{
|
||||
driver.get("https:"+m.url)
|
||||
Thread.sleep(3000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def obj = parsering(doc, m.url)
|
||||
if(obj){
|
||||
obj.proince = proince
|
||||
obj.city = city
|
||||
results << obj
|
||||
}
|
||||
// }catch(Exception e){
|
||||
// println m.url+" 解析有问题!"
|
||||
// }
|
||||
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
def parsering(doc, url){
|
||||
def result = [:]
|
||||
if(doc) {
|
||||
|
||||
def title = JQueryUtils.selectText([".d-m-title"], doc)
|
||||
if(!title) {
|
||||
title = JQueryUtils.selectText([".DivItemName"], doc)
|
||||
@@ -48,6 +188,9 @@ class PaimaiTest {
|
||||
}
|
||||
result.title = title
|
||||
result.status = status
|
||||
if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){
|
||||
println title
|
||||
}
|
||||
|
||||
if(title) {
|
||||
def newStr =title
|
||||
@@ -76,7 +219,7 @@ class PaimaiTest {
|
||||
if(curPriceStr.contains("成交价:")) {
|
||||
def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","")
|
||||
if(tradePriceStr) {
|
||||
result.tradePrice = tradePriceStr.toInteger()
|
||||
result.tradePrice = tradePriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,23 +228,28 @@ class PaimaiTest {
|
||||
curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","")
|
||||
result.tradePrice = curPriceStr.toInteger()
|
||||
result.tradePrice = curPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc)
|
||||
if(enrollCount) {
|
||||
def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc)
|
||||
if(enrollCount && enrollCount.isNumber()) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc)
|
||||
if(callCount) {
|
||||
def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc)
|
||||
if(callCount && callCount.isNumber()) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc)
|
||||
if(lookCount && lookCount.isNumber()) {
|
||||
try{
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}catch(Exception e){
|
||||
println lookCount
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc)
|
||||
@@ -140,7 +288,7 @@ class PaimaiTest {
|
||||
}else if(str.contains("保证金")) {
|
||||
def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "")
|
||||
if(v) {
|
||||
result.depositPrice = v.toInteger()
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}
|
||||
}else if(str.contains("延时周期")) {
|
||||
|
||||
@@ -184,6 +332,47 @@ class PaimaiTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(detailMap && !detailMap.get("拍品名称")){
|
||||
detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
|
||||
if(tr.text().contains("拍品名称")){
|
||||
def td1 = tr.select("td:contains(拍品名称)+td").text()
|
||||
detailMap.put("拍品名称", td1)
|
||||
}
|
||||
|
||||
if(tr.text().contains("权利来源")){
|
||||
def td1 = tr.select("td:contains(权利来源)+td").text()
|
||||
detailMap.put("权利来源", td1)
|
||||
}
|
||||
|
||||
if(tr.text().contains("权证情况")){
|
||||
def td1 = tr.select("td:contains(权证情况)+td").text()
|
||||
detailMap.put("权证情况", td1)
|
||||
}
|
||||
|
||||
if(tr.text().contains("拍品所有人")){
|
||||
def td1 = tr.select("td:contains(拍品所有人)+td").text()
|
||||
detailMap.put("拍品所有人", td1)
|
||||
}
|
||||
|
||||
if(tr.text().contains("房屋情况")){
|
||||
def td1 = tr.select("td:contains(房屋情况)+td").text()
|
||||
detailMap.put("房屋情况", td1)
|
||||
}
|
||||
|
||||
if(tr.text().contains("权利限制情况及瑕疵情况")){
|
||||
def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text()
|
||||
detailMap.put("权利限制情况及瑕疵情况", td1)
|
||||
}
|
||||
|
||||
if(tr.text().contains("拍品介绍")){
|
||||
def td1 = tr.select("td:contains(拍品介绍)").text()
|
||||
detailMap.put("拍品介绍", td1)
|
||||
}
|
||||
}
|
||||
}
|
||||
result.detailMap = detailMap
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
@@ -248,17 +437,18 @@ class PaimaiTest {
|
||||
}
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(s)
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
|
||||
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
|
||||
if(!el) {
|
||||
el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc)
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(s)
|
||||
// result.content = DataUtil.zip(s)
|
||||
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
|
||||
result.url = url
|
||||
result.mDate = new Date();
|
||||
return result;
|
||||
}
|
||||
|
||||
def find(url){
|
||||
@@ -374,4 +564,32 @@ class PaimaiTest {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveData(def results){
|
||||
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
obj.source = "公拍网"
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+348
-71
@@ -1,12 +1,4 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.gj.paimai.PaimaiDataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -14,66 +6,266 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.JavascriptExecutor
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
import org.openqa.selenium.support.ui.ExpectedCondition
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
|
||||
|
||||
def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get(url);
|
||||
driver.findElementById("1").click()
|
||||
run(url, driver)
|
||||
//driver.get(url);
|
||||
//driver.findElementById("1").click()
|
||||
//driver.findElementById("search_sub").click()
|
||||
//
|
||||
//def provinces = getAllProvinceValue(driver.getPageSource())
|
||||
//for(def m : provinces){
|
||||
// def city_setting = driver.findElementById("city")
|
||||
// //${m.province_id}
|
||||
// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
|
||||
//
|
||||
////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
|
||||
////WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
|
||||
// Thread.sleep(10000)
|
||||
// def doc = Jsoup.parse(driver.getPageSource());
|
||||
// def citys = getAllCity(doc)
|
||||
// getDetailByCity(m.province, citys, driver)
|
||||
//}
|
||||
|
||||
def city_setting = driver.findElementById("city")
|
||||
city_setting.findElement(By.xpath("//option[@value='31']")).click()
|
||||
|
||||
driver.findElementById("search_sub").click()
|
||||
//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
|
||||
//WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def links = getLinks(doc)
|
||||
def run(url, driver){
|
||||
def values = getAllCityRegionValue(url, driver)
|
||||
def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"]
|
||||
for(def m : values){
|
||||
if(!finishCity.contains(m.city)){
|
||||
driver.get(url);
|
||||
driver.findElementById("1").click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
def pro_setting = driver.findElementById("city")
|
||||
pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
def city_setting = driver.findElementById("city1")
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
|
||||
Thread.sleep(4000)
|
||||
|
||||
def region_setting = driver.findElementById("city2")
|
||||
region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click()
|
||||
|
||||
driver.findElementById("search_sub").click()
|
||||
Thread.sleep(10000)
|
||||
getDetailByPage(m.province, m.city, m.region, driver)
|
||||
}
|
||||
|
||||
def eles = driver.findElementsByClassName("pagecur1")
|
||||
for(def ele : eles){
|
||||
ele.click();
|
||||
Thread.sleep(10000)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
links.addAll(getLinks(doc))
|
||||
}
|
||||
def rets = []
|
||||
for(def durl : links){
|
||||
driver.get(durl);
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def res = parse(durl, doc);
|
||||
if(res){
|
||||
rets << res
|
||||
}
|
||||
}
|
||||
saveToDB(rets);
|
||||
|
||||
def getAllCityRegionValue(def url, def driver){
|
||||
driver.get(url);
|
||||
driver.findElementById("1").click()
|
||||
driver.findElementById("search_sub").click()
|
||||
|
||||
def provinces = getAllProvinceValue(driver.getPageSource())
|
||||
def values = []
|
||||
for(def m : provinces){
|
||||
def city_setting = driver.findElementById("city")
|
||||
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
|
||||
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def citys = getAllCity(doc)
|
||||
def cityRegions = getAllCityByRegion(m, citys, driver)
|
||||
values.addAll(cityRegions)
|
||||
//getDetailByCity(m.province, citys, driver)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
def getDetailByCity(province, citys, driver){
|
||||
for(def m : citys){
|
||||
def city_setting = driver.findElementById("city1")
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = getAllRegion(doc)
|
||||
getDetailByRegion(province, m.city, regions, driver)
|
||||
}
|
||||
}
|
||||
|
||||
def getAllCityByRegion(provinceMap, citys, driver){
|
||||
def datas = []
|
||||
for(def m : citys){
|
||||
def city_setting = driver.findElementById("city1")
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = getAllRegion(provinceMap, m, doc)
|
||||
datas.addAll(regions)
|
||||
}
|
||||
return datas
|
||||
}
|
||||
|
||||
def getDetailByRegion(province, city, regions, driver){
|
||||
def links = []
|
||||
for(def r : regions){
|
||||
def city_setting = driver.findElementById("city2")
|
||||
city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click()
|
||||
driver.findElementById("search_sub").click()
|
||||
Thread.sleep(10000)
|
||||
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
links.addAll(getLinks(r.region, doc))
|
||||
def eles = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(eles){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
Thread.sleep(3000)
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
def datas = parseDatas(province, city, links, driver)
|
||||
saveToDB(datas);
|
||||
println r.region +" "+ links.size()
|
||||
}
|
||||
|
||||
def getDetailByPage(province, city, region, driver){
|
||||
def links = []
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
links.addAll(getLinks(doc))
|
||||
def eles = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(eles){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
Thread.sleep(3000)
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
def datas = parseDatas(province, city, region, links, driver)
|
||||
println province +" "+city+" "+region +" "+ links.size()
|
||||
saveToDB(datas);
|
||||
}
|
||||
|
||||
|
||||
def getAllProvinceValue(doc){
|
||||
def optionsEls = JQueryUtils.select("#city option", doc)
|
||||
def rets = []
|
||||
for(def eles : optionsEls){
|
||||
def val = eles.attr("value")
|
||||
if(val!="" && val!="0"){
|
||||
def province = eles.text()
|
||||
rets << ["province":province, "province_id":val]
|
||||
|
||||
def parse(url, doc){
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getAllCity(doc){
|
||||
def optionsEls = JQueryUtils.select("#city1 option", doc)
|
||||
def rets = []
|
||||
for(def eles : optionsEls){
|
||||
def val = eles.attr("value")
|
||||
if(val!=""){
|
||||
def province = eles.text()
|
||||
rets << ["city":province, "city_id":val]
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getAllRegion(provinceMap, cityMap,doc){
|
||||
def optionsEls = JQueryUtils.select("#city2 option", doc)
|
||||
def rets = []
|
||||
for(def eles : optionsEls){
|
||||
def val = eles.attr("value")
|
||||
if(val!=""){
|
||||
def region = eles.text()
|
||||
def regionMap = [:]
|
||||
regionMap.province = provinceMap.province
|
||||
regionMap.province_id = provinceMap.province_id
|
||||
regionMap.city = cityMap.city
|
||||
regionMap.city_id = cityMap.city_id
|
||||
regionMap.region = region
|
||||
regionMap.region_id = val
|
||||
rets << regionMap
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def parseDatas(province, city, region,links, driver){
|
||||
def datas = []
|
||||
for(def url: links){
|
||||
def doc = getDocument(url, driver)
|
||||
def result = parse(city, region, url, doc)
|
||||
if(result){
|
||||
result.province = province
|
||||
result.city = city
|
||||
result.region = region
|
||||
datas << result
|
||||
}
|
||||
}
|
||||
return datas
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parse(city, region, url, doc){
|
||||
def result=[:]
|
||||
if(doc) {
|
||||
|
||||
def title = JQueryUtils.selectText(["#Title"], doc)
|
||||
result.title = title;
|
||||
result.address = title
|
||||
if(result.address){
|
||||
def rMap = OpenApiUtils.matchDistrict("ÉϺ£",result.address)
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
|
||||
if(rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
@@ -99,6 +291,7 @@ def parse(url, doc){
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".bmnumber"], doc)
|
||||
if(enrollCount) {
|
||||
enrollCount = enrollCount.replace("人报名","")
|
||||
if(enrollCount.isInteger()){
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
@@ -107,6 +300,7 @@ def parse(url, doc){
|
||||
|
||||
def callCount = JQueryUtils.selectText([".sc"], doc)
|
||||
if(callCount) {
|
||||
callCount =callCount.replace("人收藏","")
|
||||
if(callCount.isInteger()){
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
@@ -114,6 +308,7 @@ def parse(url, doc){
|
||||
|
||||
def lookCount = JQueryUtils.selectText([".hits"], doc)
|
||||
if(lookCount) {
|
||||
lookCount = lookCount.replace("次围观","")
|
||||
if(lookCount.isInteger()){
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
@@ -123,8 +318,8 @@ def parse(url, doc){
|
||||
def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc)
|
||||
if(curPriceStr && curPriceStr.contains("起拍价")) {
|
||||
curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","")
|
||||
if(curPriceStr.contains("ÍòÔª")){
|
||||
curPriceStr = curPriceStr.replace("ÍòÔª","")
|
||||
if(curPriceStr.contains("万")){
|
||||
curPriceStr = curPriceStr.replace("万元","").replace("万","")
|
||||
if(curPriceStr.isInteger()){
|
||||
result.startPrice = curPriceStr.toInteger()*10000
|
||||
}else{
|
||||
@@ -144,6 +339,44 @@ def parse(url, doc){
|
||||
}
|
||||
}
|
||||
|
||||
def tableEles = JQueryUtils.select("#bg1", doc)
|
||||
if(tableEles){
|
||||
def evaluatePriceStr = tableEles.select("td:contains(值:)").text()
|
||||
if(evaluatePriceStr && evaluatePriceStr.contains(":")){
|
||||
def arrs = evaluatePriceStr.split(":")
|
||||
if(arrs.size()==2){
|
||||
evaluatePriceStr = arrs[1]
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceStr = tableEles.select("td:contains(金:)").text()
|
||||
if(depositPriceStr && depositPriceStr.contains(":")){
|
||||
def arrs = depositPriceStr.split(":")
|
||||
if(arrs.size()==2){
|
||||
depositPriceStr = arrs[1]
|
||||
}
|
||||
}
|
||||
if(depositPriceStr){
|
||||
depositPriceStr = depositPriceStr.replace("万元","")
|
||||
if(depositPriceStr.toString().isNumber()){
|
||||
result.depositPrice = depositPriceStr.toDouble()*10000
|
||||
}
|
||||
}
|
||||
|
||||
def courtStr = tableEles.select("td:contains(院:)").text()
|
||||
if(courtStr && courtStr.contains(":")){
|
||||
result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size())
|
||||
}
|
||||
def personStr = tableEles.select("td:contains(人:)").text()
|
||||
if(personStr && personStr.contains(":")){
|
||||
result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size())
|
||||
}
|
||||
|
||||
def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","")
|
||||
if(countStr && countStr.contains("第") && countStr.contains("次")){
|
||||
result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次"))
|
||||
}
|
||||
}
|
||||
|
||||
def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc)
|
||||
if(lis && lis.size()) {
|
||||
@@ -151,18 +384,23 @@ def parse(url, doc){
|
||||
def str = li.select("span").text()
|
||||
if(str.contains("评估值")) {
|
||||
def v = li.select("span:eq(1)").text().replace("万元", "")
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toInteger()*10000
|
||||
}else{
|
||||
result.evaluationPrice = (int)(v.toDouble()*10000)
|
||||
if(v.isNumber()){
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toInteger()*10000
|
||||
}else{
|
||||
result.evaluationPrice = (int)(v.toDouble()*10000)
|
||||
}
|
||||
}
|
||||
|
||||
}else if(str.contains("保证金")) {
|
||||
def v = li.select("span:eq(1)").text().replace("万元", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toInteger()*10000
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble()*10000)
|
||||
if(v.isNumber()) {
|
||||
if (v) {
|
||||
if (v.isInteger()) {
|
||||
result.depositPrice = v.toInteger() * 10000
|
||||
} else {
|
||||
result.depositPrice = (int) (v.toDouble() * 10000)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if (str.contains("公告日期:")){
|
||||
@@ -177,13 +415,32 @@ def parse(url, doc){
|
||||
def trs = tables[1].select("tr")
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(tr.text().contains("调查情况表")){
|
||||
continue
|
||||
}
|
||||
if(d_tds.size()>1){
|
||||
def lastEleStr = d_tds[d_tds.size()-1].toString()
|
||||
if(lastEleStr.contains('height="42"')){
|
||||
d_tds.remove(d_tds[d_tds.size()-1])
|
||||
}
|
||||
}
|
||||
|
||||
if(d_tds.size()>3){
|
||||
continue
|
||||
}
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
if(d_tds[1].text() && d_tds[2].text()){
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}
|
||||
|
||||
}else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
if(d_tds[0].text() && d_tds[1].text()){
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}
|
||||
}else {
|
||||
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
@@ -202,8 +459,6 @@ def parse(url, doc){
|
||||
result.introduce = tables[1].toString()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def imgs =JQueryUtils.select("#bdjs11 img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
@@ -232,6 +487,20 @@ def parse(url, doc){
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText(["#pmgg"], doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = PaimaiDataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
|
||||
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
|
||||
}
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
|
||||
result.url = url
|
||||
@@ -246,7 +515,7 @@ def getLinks(doc){
|
||||
if(eles){
|
||||
for(def el :eles){
|
||||
def href = el.attr("href")
|
||||
urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
|
||||
urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -264,13 +533,19 @@ def saveToDB(rets){
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.remove("url")
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
paimaiResultColl.save(obj)
|
||||
}
|
||||
obj.source= "人民法院诉讼资产网"
|
||||
try{
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.save(obj)
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error:"+obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) {
|
||||
|
||||
for (def val : list) {
|
||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
if(v && v.isNumber()){
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+416
-34
@@ -1,7 +1,12 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class TaobaoPaimaiCrawl {
|
||||
static main(args) {
|
||||
@@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1"
|
||||
def regionUrls = regionCrawl(seedUrl);
|
||||
def pageUrls = pageUrlCrawl(regionUrls)
|
||||
detailCrawl(pageUrls)
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1"
|
||||
def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
getAllCityRegions(driver, url);
|
||||
}
|
||||
|
||||
def regionCrawl(def seedUrl){
|
||||
def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk");
|
||||
def regionUrls = []
|
||||
if(document){
|
||||
def eles = JQueryUtils.select(".small-subcondion a", document)
|
||||
if(eles){
|
||||
for(def el : eles){
|
||||
def reiongUrl = el.attr("href").trim()
|
||||
if(reiongUrl.startsWith("//")){
|
||||
regionUrls << "http:"+reiongUrl
|
||||
def getAllCityRegions(driver, url){
|
||||
|
||||
def provinces = getAllProvince(driver, url)
|
||||
def citys = ["天津","上海","北京","重庆"]
|
||||
|
||||
for(def province : provinces){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles= JQueryUtils.select(".unfold .J_FilterCity a")
|
||||
for(def cityEle : cityEles){
|
||||
def city = cityEle.text()
|
||||
if(citys.contains(city)){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
|
||||
driver.findElement(By.cssSelector(".J_FilterCity a")).click()
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a")
|
||||
for(def r : regionEles){
|
||||
def m = [:]
|
||||
m.province = province
|
||||
m.city = city
|
||||
m.region = r.text()
|
||||
def urls = crawlLinks(m, driver)
|
||||
parse(m, driver, urls)
|
||||
}
|
||||
}else{
|
||||
def m = [:]
|
||||
m.province = province
|
||||
m.city = city
|
||||
def urls = crawlLinks(m, driver)
|
||||
parse(m, driver, urls)
|
||||
}
|
||||
}
|
||||
}
|
||||
return regionUrls;
|
||||
}
|
||||
|
||||
def pageUrlCrawl(regionUrls){
|
||||
def crawlLinks(def m, def driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
for(def url : regionUrls){
|
||||
def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk");
|
||||
if(document){
|
||||
def pages = JQueryUtils.selectText(".page-total",document)
|
||||
if(pages){
|
||||
pages = pages.toInteger()
|
||||
for(def page=1;page<pages;page++){
|
||||
urls << url+"&page="+page
|
||||
while(!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def details = JQueryUtils.select(".sf-pai-item-list li a", doc)
|
||||
|
||||
for(def d : details){
|
||||
urls << "https:"+d.attr("href")
|
||||
}
|
||||
def nextEle = JQueryUtils.select(".next", doc)
|
||||
if(nextEle){
|
||||
driver.findElement(By.cssSelector(".next")).click()
|
||||
}
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
def getAllProvince(driver, url){
|
||||
driver.get(url);
|
||||
Thread.sleep(100000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles= JQueryUtils.select("li:contains(标的物类型) .condition li a", doc)
|
||||
def provinces = []
|
||||
for(def e : eles){
|
||||
def province = e.text()
|
||||
provinces << province
|
||||
}
|
||||
return provinces
|
||||
}
|
||||
|
||||
def parse(m, driver, urls){
|
||||
def results = []
|
||||
for(def url : urls) {
|
||||
driver.get(url);
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def result = [:]
|
||||
if (doc) {
|
||||
|
||||
def title = JQueryUtils.selectText([".pm-main h1"], doc)
|
||||
def statusDesc = JQueryUtils.selectText([".bid-fail"], doc)
|
||||
|
||||
result.province = m.province
|
||||
result.city = m.city
|
||||
if (m.region) {
|
||||
result.region = m.region
|
||||
}
|
||||
if (title) {
|
||||
|
||||
def addr = title.toString().replace("本场拍卖已", "").replace(" !", "")
|
||||
if (addr && addr.indexOf("室") > -1) {
|
||||
result.address = addr.substring(0, addr.indexOf("室") + 1)
|
||||
} else {
|
||||
result.address = addr
|
||||
}
|
||||
|
||||
if (result.city) {
|
||||
if (result.address.contains(city + "市")) {
|
||||
result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1)
|
||||
} else {
|
||||
result.address = result.address.substring(result.address.indexOf(city) + city.length())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (result.address.indexOf("】") > -1) {
|
||||
result.address = result.address.substring(result.address.indexOf("】") + 1)
|
||||
}
|
||||
|
||||
|
||||
if (result.city) {
|
||||
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
}
|
||||
if (statusDesc) {
|
||||
def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1)
|
||||
result.status = DataUtil.getPaiMaiType(status)
|
||||
result.statusDesc = statusDesc
|
||||
}
|
||||
|
||||
def statusEle = JQueryUtils.select(".status-tip", doc)
|
||||
if (statusEle) {
|
||||
def sEle = JQueryUtils.select(".status-ing", doc)
|
||||
if (sEle) {
|
||||
result.status = "进行中"
|
||||
} else {
|
||||
result.status = "即将开始"
|
||||
}
|
||||
}
|
||||
def curPrice = JQueryUtils.selectText(["#sf-price"], doc)
|
||||
if (curPrice) {
|
||||
if (curPrice.contains("当前价")) {
|
||||
curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
|
||||
if (curPrice) {
|
||||
curPrice = curPrice.replace(",", "")
|
||||
if (curPrice.isInteger()) {
|
||||
result.curPrice = curPrice.toInteger()
|
||||
} else {
|
||||
result.curPrice = (int) (curPrice.toDouble())
|
||||
}
|
||||
}
|
||||
} else if (curPrice.contains("成交价")) {
|
||||
def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
|
||||
if (tradePrice) {
|
||||
tradePrice = tradePrice.replace(",", "")
|
||||
if (tradePrice.isInteger()) {
|
||||
result.tradePrice = tradePrice.toInteger()
|
||||
} else {
|
||||
result.tradePrice = (int) (tradePrice.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
} else if (curPrice.contains("起拍价")) {
|
||||
def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
|
||||
if (startPrice) {
|
||||
startPrice = startPrice.replace(",", "")
|
||||
if (startPrice.isInteger()) {
|
||||
result.startPrice = startPrice.toInteger()
|
||||
} else {
|
||||
result.startPrice = (int) (startPrice.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc)
|
||||
if (finishStr) {
|
||||
if (finishStr.contains("结束时间")) {
|
||||
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
|
||||
if (finishStr) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
|
||||
}
|
||||
} else if (finishStr.contains("成交时间")) {
|
||||
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
|
||||
if (finishStr) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc)
|
||||
if (enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc)
|
||||
if (callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText(["#J_Looker"], doc)
|
||||
if (lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
def tds = JQueryUtils.select(["#J_HoverShow td"], doc)
|
||||
if (tds && tds.size() > 0) {
|
||||
for (def td : tds) {
|
||||
def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "")
|
||||
if (str) {
|
||||
if (str.contains("变卖价")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.sellOffPrice = p.toInteger()
|
||||
} else {
|
||||
result.sellOffPrice = (int) (p.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
} else if (str.contains("加价幅度")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.addPrice = p.toInteger()
|
||||
} else {
|
||||
result.addPrice = (int) (p.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
} else if (str.contains("类型")) {
|
||||
|
||||
result.type = td.select(".pay-type").text()
|
||||
|
||||
} else if (str.contains("保证金")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.depositPrice = p.toInteger()
|
||||
} else {
|
||||
result.depositPrice = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
} else if (str.contains("变卖周期")) {
|
||||
|
||||
result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("竞价周期")) {
|
||||
|
||||
result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("优先购买权")) {
|
||||
|
||||
result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("变卖预缴款")) {
|
||||
def str1 = td.select("span:eq(1) .J_Price").text()
|
||||
def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.sellOffAdvancePay = p.toInteger()
|
||||
} else {
|
||||
result.sellOffAdvancePay = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
} else if (str.contains("评估价")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.evaluationPrice = p.toInteger()
|
||||
} else {
|
||||
result.evaluationPrice = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
} else if (str.contains("延时周期")) {
|
||||
|
||||
result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("起拍价")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.startPrice = p.toInteger()
|
||||
} else {
|
||||
result.startPrice = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc)
|
||||
|
||||
def trs = JQueryUtils.select(["#J_desc table tr"], doc)
|
||||
if (trs) {
|
||||
def detailMap = [:]
|
||||
for (def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if (d_tds && d_tds.size() >= 2) {
|
||||
if (d_tds.size() == 3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
} else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
} else {
|
||||
if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
|
||||
if (!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if (!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def imgs = JQueryUtils.select([".slide-bigpic img"], doc)
|
||||
if (imgs) {
|
||||
def imgLinks = []
|
||||
for (def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if (href.contains("http")) {
|
||||
imgLinks << href
|
||||
} else {
|
||||
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
|
||||
}
|
||||
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
}
|
||||
|
||||
//面积
|
||||
def article = JQueryUtils.selectText(["#J_ItemNotice"], doc)
|
||||
if (article) {
|
||||
def area = DataUtil.getAreaNumber(article, 5000)
|
||||
if (!area) {
|
||||
article = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
|
||||
if (article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
if (!area) {
|
||||
article = JQueryUtils.selectText(["body"], doc)
|
||||
if (article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
if (area) {
|
||||
result.area = area.toDouble()
|
||||
if (result.tradePrice) {
|
||||
result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def detailCrawl(pageUrls){
|
||||
if(pageUrls){
|
||||
for(def pageUrl : pageUrls){
|
||||
def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk");
|
||||
if(document){
|
||||
def pages = JQueryUtils.select(".page-total", document)
|
||||
|
||||
def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
|
||||
if (s1) {
|
||||
def dateRangs = DataUtil.getDateRange(s1)
|
||||
if (dateRangs) {
|
||||
for (def j = 0; j < dateRangs.size(); j++) {
|
||||
def date = dateRangs[j]
|
||||
def pattern = null
|
||||
if (date.endsWith("日时") || date.endsWith("日")) {
|
||||
pattern = "yyyy年MM月dd日"
|
||||
} else {
|
||||
|
||||
def ind = date.indexOf("日")
|
||||
if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) {
|
||||
date = date + "时"
|
||||
}
|
||||
|
||||
pattern = "yyyy年MM月dd日HH时"
|
||||
}
|
||||
if (j == 0) {
|
||||
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
} else {
|
||||
if (!result.finishDate) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.startPrice) {
|
||||
result.startPrice = DataUtil.getStartPrice(s1)
|
||||
}
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
def estateType = DataUtil.findEstateType(s)
|
||||
if (estateType) {
|
||||
result.matchEstateType = estateType
|
||||
}
|
||||
|
||||
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
|
||||
result.url = url
|
||||
result.mDate = new Date()
|
||||
results << result
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
|
||||
+7
-2
@@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
|
||||
class BlankEvaluationBatch {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
class EvaluationBatch {
|
||||
static main(args) {
|
||||
def task = new EvaluationBatch();
|
||||
def task = new BlankEvaluationBatch();
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.gj.price
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
class FzMatchTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FzMatchTest();
|
||||
// task.run()
|
||||
task.caluatePPE()
|
||||
}
|
||||
|
||||
def caluatePPE(){
|
||||
def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区_结果1).xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { data ->
|
||||
if(data['F'] && data['G']){
|
||||
def dist = CommonDatautils.findDistrict("福州", data['F'], data['G'], "district")
|
||||
if(dist){
|
||||
def avg_price = findAvgPrice(dist._id.toString())
|
||||
if(data['i'] && avg_price){
|
||||
def weekendPrice = data['i'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
data['J'] = ppe15
|
||||
}
|
||||
}
|
||||
}
|
||||
},1)
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区).xlsx"
|
||||
def city = "福州"
|
||||
ExcelUtils.multiThreadingModify(filePath, { data ->
|
||||
// if(data['G']){
|
||||
// if(!data['H']){
|
||||
// def dist = CommonDatautils.findDistrict(city, data['F'], data['G'], "district")
|
||||
//
|
||||
// if(dist && dist.finish_building_date){
|
||||
// data['H'] = dist.finish_building_date.substring(0,4)
|
||||
// }
|
||||
// }
|
||||
// if(!data['I']){
|
||||
// def price = CommonDatautils.queryDistrictPrice(city, data['F'], data['G'])
|
||||
// if(price){
|
||||
// data['I'] = price
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
if(!data['G'] && data['B']){
|
||||
def name = data['B'].replace("(", "").replace(")","").replace("\\","").replace("#","")
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, null, name, null);
|
||||
if(rMap) {
|
||||
data['F'] = rMap.region
|
||||
data['G'] = rMap.name
|
||||
def dist = CommonDatautils.findDistrictById(rMap.id)
|
||||
if(dist && dist.finish_building_date){
|
||||
data['H'] = dist.finish_building_date.substring(0,4)
|
||||
}
|
||||
|
||||
def price = CommonDatautils.queryDistrictPrice(city, rMap.region, rMap.name)
|
||||
if(price){
|
||||
data['I'] = price
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 1);
|
||||
}
|
||||
}
|
||||
+62
-24
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
def source = "58同城"
|
||||
def root = "58.com"
|
||||
def shortCityMap = findCitySeeds(root)
|
||||
def datas = findCrawlData(source, shortCityMap, root)
|
||||
def list = []
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def findCitySeeds(root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("root", root)
|
||||
q.put("city", "福州")
|
||||
|
||||
def dbcursor = districtseedsColl.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(!cityMap.keySet().contains(d.city)){
|
||||
cityMap.put(d.city, d.city_short)
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
}
|
||||
|
||||
def findCrawlData(def source, shortCityMap, def root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","¸£ÖÝ")
|
||||
q.put("source", "³ÇÊз¿²ú")
|
||||
q.put("s_date",202012)
|
||||
q.put("source", source)
|
||||
q.put("s_date",202112)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = ".cityhouse.cn/ha/"
|
||||
def prefix = "xiaoqu/"
|
||||
def keyList =new HashSet()
|
||||
def index = 0
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
@@ -76,33 +96,42 @@ class CrawlDistrictIdSelect {
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
|
||||
|
||||
if(url){
|
||||
def district_id = ""
|
||||
if(url.contains(prefix)){
|
||||
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
|
||||
}else{
|
||||
district_id = url;
|
||||
}
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
def ids = retMap.get(db.city)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
retMap.put(db.city, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "cityhouse.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
if(!keyList.contains(db.city+"#"+district_id)){
|
||||
def count = findIsExists(db.city, district_id, root)
|
||||
if(!count){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "58.com"
|
||||
map.city_short= shortCityMap.get(db.city)
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
map.name = db.name
|
||||
ids << map
|
||||
}else{
|
||||
index++
|
||||
println "[${index}]已存在:"+db.city +" "+db.name
|
||||
}
|
||||
keyList << db.city+"#"+district_id
|
||||
}else{
|
||||
println "ÖØ¸´£º"+key
|
||||
println "重复:"+db.city+"#"+db.name
|
||||
}
|
||||
}else{
|
||||
println url +" Á´½Ó²»ºÏÒªÇó"
|
||||
@@ -110,4 +139,13 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findIsExists(city, district_id, root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("district_id", district_id)
|
||||
q.put("root",root)
|
||||
def count = districtseedsColl.count(q)
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -68,7 +68,7 @@ class FZloationUpdate {
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ class FZloationUpdate {
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
if(reps && reps.count && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
|
||||
+50
-1
@@ -42,6 +42,55 @@ class SFTP {
|
||||
return sftp;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 删除本地文件
|
||||
*
|
||||
* @param filePath
|
||||
* @return
|
||||
*/
|
||||
public boolean deleteFile(String filePath) {
|
||||
File file = new File(filePath);
|
||||
if (!file.exists()) {
|
||||
return false;
|
||||
}
|
||||
if (!file.isFile()) {
|
||||
file.deleteDir();
|
||||
return false;
|
||||
}
|
||||
boolean rs = file.delete();
|
||||
return rs;
|
||||
}
|
||||
|
||||
public static void deleteSFTP(String directory, ChannelSftp sftp) {
|
||||
try {
|
||||
if (isDirExist(directory, sftp)) {
|
||||
Vector<ChannelSftp.LsEntry> vector = sftp.ls(directory);
|
||||
if (vector.size() == 1) { // 文件,直接删除
|
||||
sftp.rm(directory);
|
||||
} else if (vector.size() == 2) { // 空文件夹,直接删除
|
||||
sftp.rmdir(directory);
|
||||
} else {
|
||||
String fileName = "";
|
||||
// 删除文件夹下所有文件
|
||||
for (ChannelSftp.LsEntry en : vector) {
|
||||
fileName = en.getFilename();
|
||||
if (".".equals(fileName) || "..".equals(fileName)) {
|
||||
continue;
|
||||
} else {
|
||||
deleteSFTP(directory + "/" + fileName);
|
||||
}
|
||||
}
|
||||
// 删除文件夹
|
||||
sftp.rmdir(directory);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传文件
|
||||
* @param directory 上传的目录
|
||||
@@ -132,7 +181,7 @@ class SFTP {
|
||||
sftp.cd(filePath.toString());
|
||||
} else {
|
||||
// 建立目录
|
||||
sftp.mkdir(filePath.toString());
|
||||
sftp.mkdir(filePath.toString())
|
||||
// 进入并设置为当前目录
|
||||
sftp.cd(filePath.toString());
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");//·¢ËÍ·½
|
||||
email.setSubject(mail.subject); //±êÌâ
|
||||
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");
|
||||
email.setSubject(mail.subject);
|
||||
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class CqfdcChromeCrawl {
|
||||
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new CqfdcChromeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def list = getListDatas(driver)
|
||||
getDetails(driver, list);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getDetails(driver, list){
|
||||
def index = 0
|
||||
for(def map_old : list){
|
||||
def datas = []
|
||||
index++
|
||||
def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}"
|
||||
// driver.get(url)
|
||||
// def doc = driver.getPageSource()
|
||||
def cityDoc = getDocment(url, driver)
|
||||
def trs = JQueryUtils.select(".building_table tr", cityDoc)
|
||||
def unit1 = null
|
||||
def unit2 = null
|
||||
def unit3 = null
|
||||
def unit1_text = null
|
||||
def unit2_text = null
|
||||
def unit3_text = null
|
||||
for(def i=0;i<trs.size();i++){
|
||||
if(i==1){
|
||||
continue
|
||||
}
|
||||
def tr = trs.get(i)
|
||||
def tds = tr.select("td")
|
||||
for(def j=2;j< tds.size();j++){
|
||||
def td = tds[j]
|
||||
if(i==0){
|
||||
def text = td.text()
|
||||
|
||||
if(!unit1){
|
||||
if(text && text.contains("单元")){
|
||||
unit1 = td.attr("colspan").toInteger()
|
||||
unit1_text = text.replace("单元","")
|
||||
}
|
||||
}else if(unit1 && !unit2){
|
||||
if(text && text.contains("单元")) {
|
||||
unit2 = td.attr("colspan").toInteger()
|
||||
unit2_text = text.replace("单元", "")
|
||||
}
|
||||
}else if(unit1 && !unit3){
|
||||
if(text && text.contains("单元")) {
|
||||
unit3 = td.attr("colspan").toInteger()
|
||||
unit3_text = text.replace("单元", "")
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def room_num = td.select("a font").text()
|
||||
if(room_num){
|
||||
def map = [:]
|
||||
def a = td.select("a")
|
||||
map.use_type = td.select("span:contains(用途:)").text().replace("用途:","")
|
||||
map.room_type = td.select("span:contains(户型:)").text().replace("户型:","")
|
||||
map.structure_type = td.select("span:contains(结构:)").text().replace("结构:","")
|
||||
map.area = td.select("span:contains(建面:)").text().replace("建面:","")
|
||||
map.siji_area = td.select("span:contains(套内:)").text().replace("套内:","")
|
||||
map.price = td.select("span:contains(建面拟售价格:)").text().replace("建面拟售价格:","")
|
||||
map.siji_price = td.select("span:contains(套内拟售价格:)").text().replace("套内拟售价格:","")
|
||||
map.fitment = td.select("span:contains(装修水平:)").text().replace("装修水平:","").replace("null","")
|
||||
def physical_floor = tds[0]//物理层
|
||||
def nominal_floor = tds[1]//名义层
|
||||
map.nominal_floor = nominal_floor.text()
|
||||
map.physical_floor = physical_floor.text()
|
||||
|
||||
if(unit1 && (j+1 <= unit1+2)){
|
||||
map.unit_num = unit1_text
|
||||
}else if(unit2 && (j+1 <= unit1+unit2+2)){
|
||||
map.unit_num = unit2_text
|
||||
}else if(unit3 && (j+1 <= unit1+unit2+unit3+2)){
|
||||
map.unit_num = unit3_text
|
||||
}
|
||||
map.room_num = room_num
|
||||
map_old.each{k,v->
|
||||
map.put(k,v)
|
||||
}
|
||||
datas << map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
saveDatas(datas)
|
||||
println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size()
|
||||
}
|
||||
}
|
||||
|
||||
def getDocment(def url, def driver){
|
||||
def stop = false
|
||||
def doc = null
|
||||
def error = 0
|
||||
while (!stop || error>=10){
|
||||
try{
|
||||
driver.get(url)
|
||||
def docment = driver.getPageSource()
|
||||
if(docment){
|
||||
doc = Jsoup.parse(docment)
|
||||
stop = true;
|
||||
}else{
|
||||
error ++
|
||||
}
|
||||
}catch(Exception e){
|
||||
error ++
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
def getListDatas(driver){
|
||||
driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html")
|
||||
def datas = []
|
||||
for(def i=2;i<=208;i++){
|
||||
def docment = Jsoup.parse(driver.getPageSource());
|
||||
def trs = JQueryUtils.select("#searchbody tr", docment)
|
||||
for(def tr : trs){
|
||||
def tds = tr.select("td")
|
||||
|
||||
def hrefs = tds.get(3).select("a")
|
||||
for(def href : hrefs){
|
||||
def map = [:]
|
||||
map.house_num_desc = href.text()
|
||||
def str = href.attr("href")
|
||||
def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','"))
|
||||
map.building_id = building_id
|
||||
|
||||
map.city = "重庆"
|
||||
map.name = tds.get(0).text()
|
||||
map.company = tds.get(1).text()
|
||||
map.license_key = tds.get(2).text()
|
||||
datas << map
|
||||
}
|
||||
}
|
||||
println i +" "+datas.size()
|
||||
try{
|
||||
driver.findElement(By.id("nextpage")).click()
|
||||
}catch(Exception e){
|
||||
Thread.sleep(8000)
|
||||
driver.findElement(By.id("nextpage")).click()
|
||||
}
|
||||
|
||||
}
|
||||
println "列表:"+datas.size()
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def saveDatas(def datas){
|
||||
def rows = []
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d.city)
|
||||
q.put("name",d.name)
|
||||
q.put("house_num_desc", d.house_num_desc)
|
||||
q.put("room_num", d.room_num)
|
||||
q.put("unit_num",d.unit_num)
|
||||
def obj = loudongConn.findOne(q)
|
||||
if(!obj){
|
||||
def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num
|
||||
if(!keys.contains(key)){
|
||||
def udata = new BasicDBObject()
|
||||
d.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
rows << udata
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rows){
|
||||
loudongConn.insert(rows);
|
||||
println "save: "+rows.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class CqfdcMatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
|
||||
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new CqfdcMatch();
|
||||
// task.run()
|
||||
// task.stat()
|
||||
// task.export2()
|
||||
// task.matchLoudong()
|
||||
|
||||
task.addLoudong()
|
||||
}
|
||||
def export(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", new BasicDBObject('$exists', false))
|
||||
|
||||
def field = new BasicDBObject()
|
||||
field.put("city",1)
|
||||
field.put("region",1)
|
||||
field.put("district_name",1)
|
||||
field.put("address",1)
|
||||
field.put("type",1)
|
||||
def dbcursor = loudongConn.find(q, field).batchSize(5000)
|
||||
def keys = []
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","类型"]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.district_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
rets << [d.city, d.region, d.district_name, d.address, d.type]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas= findAddress2()
|
||||
def results = match(datas);
|
||||
update(results)
|
||||
}
|
||||
|
||||
|
||||
def export2(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", new BasicDBObject('$exists', false))
|
||||
|
||||
def field = new BasicDBObject()
|
||||
field.put("city",1)
|
||||
field.put("name",1)
|
||||
field.put("use_type",1)
|
||||
def dbcursor = loudongFdcConn.find(q, field).batchSize(5000)
|
||||
def keys = []
|
||||
def rets = []
|
||||
rets << ["城市","地址","类型"]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
rets << [d.city, d.name,d.use_type]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
def update(results){
|
||||
for(def m : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", m.old_city)
|
||||
q.put("district_name", m.old_name)
|
||||
q.put("region",m.old_region)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_city", m.match_city)
|
||||
udata.put("match_region", m.match_region)
|
||||
udata.put("match_name", m.match_name)
|
||||
loudongConn.update(q, new BasicDBObject('$set',udata),false, true)
|
||||
}
|
||||
}
|
||||
|
||||
def match(def datas){
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
def map = [:]
|
||||
map.match_city = jsonObject.get("city")
|
||||
map.match_region = jsonObject.get("district")
|
||||
map.match_name = jsonObject.get("name")
|
||||
map.old_name = d.name
|
||||
map.old_city = d.city
|
||||
map.old_region = d.region
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAddress(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q)
|
||||
def rets = []
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
if(!keys.contains(d.name)){
|
||||
|
||||
keys << d.name
|
||||
def map = [:]
|
||||
map.city = d.city
|
||||
map.name = d.name
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAddress2(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.district_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def map = [:]
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.district_name
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllLoundongByHouseNum(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num
|
||||
if(d.unit_num){
|
||||
key = key +"##"+d.unit_num
|
||||
}
|
||||
def detailMap= map.get(key)
|
||||
if(!detailMap){
|
||||
detailMap = [:]
|
||||
map.put(key,detailMap)
|
||||
}
|
||||
|
||||
if(detailMap.height){
|
||||
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def stat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", new BasicDBObject('$exists', true))
|
||||
|
||||
def field = new BasicDBObject()
|
||||
field.put("match_city",1)
|
||||
field.put("match_region",1)
|
||||
field.put("match_name",1)
|
||||
def dbcursor = loudongConn.find(q, field).batchSize(2000)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
|
||||
def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000)
|
||||
for(def d : dbcursor2){
|
||||
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
println "小区:"+keys.size()
|
||||
|
||||
}
|
||||
|
||||
def matchLoudong(){
|
||||
def names = findAllLoudong("重庆")
|
||||
print names.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(names.size())
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<names.size();i++){
|
||||
def key = names[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def arrs = key.split("#")
|
||||
def jsonObject = OpenApiClient.addressMatch(arrs[0], arrs[2], "property", false, arrs[1], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
|
||||
def m = [:]
|
||||
m.id = jsonObject.get("id")
|
||||
m.city = jsonObject.get("city")
|
||||
m.region = jsonObject.get("district")
|
||||
m.name = jsonObject.get("name")
|
||||
def dist = CommonDatautils.findDistrictById(m.id)
|
||||
if(dist){
|
||||
m.sid = dist.sid
|
||||
}
|
||||
retMap.put(key, m)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
println "匹配后:"+retMap.size()
|
||||
updateLoudong(retMap)
|
||||
println "修改完成!"
|
||||
}
|
||||
|
||||
def updateLoudong(retMap){
|
||||
def index = 0
|
||||
retMap.each{key,m->
|
||||
def arrs = key.split("#")
|
||||
|
||||
def q = new BasicDBObject("city", arrs[0])
|
||||
q.put('region', arrs[1]);
|
||||
q.put('name', arrs[2]);
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district_id", m.id.toString())
|
||||
udata.put("name_in_seaweed", m.name)
|
||||
udata.put("district_no", m.sid)
|
||||
HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true);
|
||||
index++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def findAllLoudong(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district_id",new BasicDBObject('$exists', false))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("name", 1)
|
||||
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
|
||||
def names = new CopyOnWriteArrayList()
|
||||
for(def db : dbcursor){
|
||||
def key = db.city +"#"+db.region+"#"+db.name
|
||||
if(!names.contains(key)){
|
||||
names << key
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def addLoudong(){
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.interactions.Actions
|
||||
import org.openqa.selenium.support.ui.ExpectedCondition
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
class HouseBookChromeCrawl {
|
||||
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
static def city = "重庆"
|
||||
static main(args) {
|
||||
def task = new HouseBookChromeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def listPageUrl = getIndexPage(driver)
|
||||
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
|
||||
// keywords.remove("城南家园")
|
||||
// keywords.remove("康居西城")
|
||||
|
||||
println city +"共 "+keywords.size()+" 个小区"
|
||||
getDistrictIndex(driver, keywords)
|
||||
|
||||
}
|
||||
|
||||
def getListPage(driver, url){
|
||||
driver.get(url)
|
||||
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource())
|
||||
def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc)
|
||||
if(total){
|
||||
def page = getTotalPage(total, 6)
|
||||
def keywords = getAllDist(driver, url, page);
|
||||
return keywords
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getDistrictIndex(driver, keywords){
|
||||
for(def word : keywords){
|
||||
def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8")
|
||||
goToDetailIndex(driver, code)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getAllDist(driver, url, page){
|
||||
def keywords = []
|
||||
for(def i=73;i<=page;i++){
|
||||
driver.get(url+"?selectPages="+i)
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def eles = JQueryUtils.select(".title-area .title-text", doc)
|
||||
for(def e : eles){
|
||||
def text = e.text()
|
||||
// if(text=="富州北郡"){
|
||||
// println i
|
||||
// }
|
||||
keywords << text
|
||||
}
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
def getTotalPage(def total, def pageSize){
|
||||
def mode = total.toInteger()/pageSize
|
||||
if(mode > mode.toInteger()){
|
||||
mode = mode.toInteger() +1
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
|
||||
|
||||
def goToDetailIndex(driver, code){
|
||||
def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind="
|
||||
driver.get(url)
|
||||
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main"))));
|
||||
def docment = Jsoup.parse(driver.getPageSource());
|
||||
def dist_name = JQueryUtils.select(".info-pname", docment).text()
|
||||
def addInfo = JQueryUtils.select(".addr-info", docment).text()
|
||||
def region = null
|
||||
if(addInfo.contains(":") && addInfo.contains("-")){
|
||||
region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-"))
|
||||
}
|
||||
|
||||
def address = JQueryUtils.select(".addr-info span", docment).text()
|
||||
|
||||
// wait.until(new ExpectedCondition<WebElement>(){
|
||||
// @Override
|
||||
// public WebElement apply(WebDriver d){
|
||||
// return d.findElement(By.linkText("[更多]"))
|
||||
// }
|
||||
// }).click()
|
||||
driver.findElement(By.linkText("[更多]")).click()
|
||||
// Thread.sleep(10)
|
||||
// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl"))));
|
||||
def lis = checkElement(driver, "#shouUl .dongli");
|
||||
|
||||
def datas = []
|
||||
if(lis){
|
||||
for(def li : lis){
|
||||
def id = li.attr("id")
|
||||
def houseNum = li.select(".title-span").text()
|
||||
def map = [:]
|
||||
map.id = id
|
||||
map.city = city
|
||||
map.region = region
|
||||
map.district_name = dist_name
|
||||
map.address = address
|
||||
map.house_num = houseNum
|
||||
map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","")
|
||||
map.type = li.select(".dong_ul2 .title-span-hbType").text()
|
||||
def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","")
|
||||
if(floor_heigth){
|
||||
map.floor_heigth = floor_heigth
|
||||
}
|
||||
def room_count = li.select(".dong_ul2 label:contains(总户数:)").text()
|
||||
if(room_count){
|
||||
room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户"))
|
||||
if(room_count){
|
||||
map.household_count = room_count
|
||||
}
|
||||
}
|
||||
datas << map
|
||||
}
|
||||
}
|
||||
|
||||
println dist_name +" 共 "+datas.size()+" 栋"
|
||||
for(def m : datas){
|
||||
try{
|
||||
driver.findElement(By.id(m.id)).click()
|
||||
|
||||
def rows = parseDetail(driver)
|
||||
println "栋divId: "+m.id +" 室:"+rows.size()
|
||||
saveDate(m, rows)
|
||||
}catch(Exception e){
|
||||
println dist_name+" 栋divId: "+m.id+" 找不到相应的div element"
|
||||
}
|
||||
|
||||
// driver.navigate().back();
|
||||
}
|
||||
}
|
||||
|
||||
def checkElement(driver, className){
|
||||
def loop = 0
|
||||
def stop = false
|
||||
def lis = null
|
||||
while (!stop){
|
||||
loop++
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
lis = JQueryUtils.select(className, doc)
|
||||
if(lis || loop>=10){
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}
|
||||
return lis
|
||||
|
||||
}
|
||||
|
||||
def parseDetail(driver){
|
||||
def trs = checkElement(driver, ".rooms tr");
|
||||
def rows = []
|
||||
for(def tr : trs){
|
||||
def tds = tr.select("td")
|
||||
if(tds){
|
||||
def floor = null
|
||||
for(def j=0;j<tds.size();j++){
|
||||
if(j == 0){
|
||||
floor = tds[0].text().replace("第","").replace("层","")
|
||||
}else {
|
||||
def td = tds[j]
|
||||
def detail = [:]
|
||||
def room_num = td.select("b").text()
|
||||
if (room_num) {
|
||||
detail.room_num = room_num
|
||||
}
|
||||
def area = td.select("span:contains(面积:)").text().replace("面积:", "").replace("㎡", "")
|
||||
if (area) {
|
||||
detail.area = area.toDouble()
|
||||
}
|
||||
def avg_price = td.select("span:contains(单价:)").text().replace("单价:", "").replace("元/㎡", "")
|
||||
if (avg_price) {
|
||||
detail.avg_price = avg_price.toInteger()
|
||||
}
|
||||
|
||||
def house_type = td.select("span:contains(房屋类型:)").text().replace("房屋类型:", "")
|
||||
if(house_type){
|
||||
detail.estate_type = house_type
|
||||
}
|
||||
if(detail){
|
||||
detail.floor = floor
|
||||
rows << detail
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
def getIndexPage(driver){
|
||||
driver.get("http://www.house-book.com.cn/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select("a:contains(新 房)", cityDoc)
|
||||
if(cityEles){
|
||||
def listPageUrl = cityEles[0].attr("href")
|
||||
return listPageUrl
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def saveDate(m, rows){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", m.region)
|
||||
q.put("district_name", m.district_name)
|
||||
def datas = []
|
||||
for(def map : rows){
|
||||
q.put("house_num", m.house_num)
|
||||
q.put("room_num", map.room_num)
|
||||
q.put("floor_heigth" , m.floor_heigth)
|
||||
q.put("floor", map.floor)
|
||||
q.put("area", map.area)
|
||||
def obj = loudongConn.findOne(q)
|
||||
if(!obj){
|
||||
def data = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
data.put(k,v)
|
||||
}
|
||||
map.each{k,v->
|
||||
data.put(k,v)
|
||||
}
|
||||
data.put("c_date", new Date())
|
||||
datas << data
|
||||
}
|
||||
}
|
||||
if(datas){
|
||||
loudongConn.insert(datas)
|
||||
}
|
||||
}
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
class HouseBookCrawl {
|
||||
|
||||
public def CHARSET = "utf-8";
|
||||
|
||||
static main(args) {
|
||||
def task = new HouseBookCrawl()
|
||||
task.run()
|
||||
}
|
||||
|
||||
// def run(){
|
||||
// goTocitypage()
|
||||
// }
|
||||
// def goTocitypage(){
|
||||
// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html"
|
||||
// def headers = [:]
|
||||
// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
|
||||
// headers.put("User-Agent", UserAgents.get())
|
||||
// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582")
|
||||
// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e")
|
||||
// def params = [:]
|
||||
// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false)
|
||||
// def cookie = getCookie(responseMap)
|
||||
// }
|
||||
//
|
||||
// def getCookie(responseMap) {
|
||||
// def cookieAuth = null
|
||||
// try {
|
||||
// def response = responseMap.get("response");
|
||||
// if (StringUtils.isNotEmpty(response)) {
|
||||
// JSONObject reJson = JSONObject.fromObject(response)
|
||||
// //登录成功获取COOKIE
|
||||
// if (reJson.success) {
|
||||
// responseMap.get("headerFields").each { k, v ->
|
||||
// if (k && k.equals("Set-Cookie")) {
|
||||
// if (v) {
|
||||
// cookieAuth = v.join(';')
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } catch (Exception e) {
|
||||
// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage()
|
||||
// }
|
||||
// println("最终构造cookieAuth为:" + cookieAuth)
|
||||
// return cookieAuth
|
||||
// }
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import net.sourceforge.tess4j.ITesseract
|
||||
import net.sourceforge.tess4j.Tesseract
|
||||
import net.sourceforge.tess4j.util.ImageHelper
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.ehcache.xml.model.TimeUnit
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.OutputType
|
||||
import org.openqa.selenium.Point
|
||||
import org.openqa.selenium.TakesScreenshot
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import javax.imageio.ImageIO
|
||||
import java.awt.image.BufferedImage
|
||||
|
||||
public class ImageTest {
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif");
|
||||
// Tesseract instance = new Tesseract();
|
||||
//
|
||||
|
||||
// //将验证码图片的内容识别为字符串
|
||||
// String result = instance.doOCR(imageFile);
|
||||
// println result
|
||||
|
||||
//1.获取网页的浏览器driver
|
||||
//(1)定义url
|
||||
String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#";
|
||||
//(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径)
|
||||
String path = System.getProperty("user.dir");
|
||||
System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
//(3)建立selenium 驱动
|
||||
//WebDriver driver = new ChromeDriver(options);
|
||||
WebDriver driver = new ChromeDriver();
|
||||
//(4)设置窗口最大化及网页等待时间
|
||||
driver.manage().window().maximize(); //窗口最大化
|
||||
// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
|
||||
//(5)启动driver浏览网页
|
||||
driver.get(url); //输入网址
|
||||
Thread.sleep(1000);
|
||||
|
||||
driver.findElement(By.id("cell1-11-2")).click()
|
||||
Thread.sleep(1000);
|
||||
|
||||
def docment = Jsoup.parse(driver.getPageSource());
|
||||
println docment
|
||||
|
||||
WebElement ele = driver.findElement(By.id("vimg"));
|
||||
|
||||
File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
|
||||
BufferedImage pic1 = ImageIO.read(screenshot);
|
||||
Point point = ele.getLocation();
|
||||
|
||||
//(2)输入验证码
|
||||
File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
|
||||
FileUtils.copyFile(srcfile, new File("picture/source.png"));
|
||||
//原图片
|
||||
//String soursePicPath = "D:\\data_dev\\test\\test.png";
|
||||
String soursePicPath = "picture/source.png";
|
||||
File sourcePic = new File(soursePicPath);
|
||||
int eleWid = ele.getSize().getWidth()
|
||||
int eleHeiht = ele.getSize().getHeight()
|
||||
|
||||
BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht);
|
||||
ImageIO.write(pic2, "jpg", sourcePic);
|
||||
|
||||
//File desImage = new File("D:\\data_dev\\test\\test02.png");
|
||||
File desImage = new File("picture/answer.png");
|
||||
|
||||
|
||||
|
||||
//加载待读取图片
|
||||
//File imageFile = new File("D:\\data_dev\\test\\test02.png");
|
||||
File imageFile = new File("picture/answer.png");
|
||||
BufferedImage image = ImageIO.read(imageFile);
|
||||
//对图片进行处理
|
||||
image = convertImage(image);
|
||||
//创建tess对象
|
||||
ITesseract instance = new Tesseract();
|
||||
//设置训练文件目录
|
||||
//instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata");
|
||||
String path02 = System.getProperty("user.dir");
|
||||
instance.setDatapath(path02 + "\\tessdata");
|
||||
//设置训练语言
|
||||
//instance.setLanguage("chi_sim");
|
||||
instance.setLanguage("eng");
|
||||
//执行转换
|
||||
String s = instance.doOCR(image);
|
||||
|
||||
System.out.println(s);
|
||||
String[] split = s.replace(" ", "").split("=|:");
|
||||
String a = split[split.length - 1].trim();
|
||||
char[] chars = a.toCharArray();
|
||||
if (chars.length == 2) {
|
||||
if (chars[0] == '7') {
|
||||
chars[0] = '1';
|
||||
}
|
||||
}
|
||||
a = String.valueOf(chars);
|
||||
|
||||
System.out.println(a);
|
||||
String b = s.substring(0, 1);
|
||||
System.out.println(b);
|
||||
String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim()));
|
||||
|
||||
|
||||
WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]"));
|
||||
input2.sendKeys(answer);
|
||||
Thread.sleep(4000);
|
||||
//(3)点击"查询"按钮
|
||||
driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click();
|
||||
|
||||
String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄
|
||||
for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历
|
||||
if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出
|
||||
driver.close();
|
||||
driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver
|
||||
System.out.println(driver.getCurrentUrl()); //打印是否为新窗口
|
||||
}
|
||||
}
|
||||
|
||||
Thread.sleep(4000);
|
||||
|
||||
//4.关闭浏览器
|
||||
driver.close();
|
||||
|
||||
}
|
||||
|
||||
|
||||
//对图片进行处理 - 提高识别度
|
||||
public static BufferedImage convertImage(BufferedImage image) throws Exception {
|
||||
//按指定宽高创建一个图像副本
|
||||
//image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight());
|
||||
//图像转换成灰度的简单方法 - 黑白处理
|
||||
image = ImageHelper.convertImageToGrayscale(image);
|
||||
//图像缩放 - 放大n倍图像
|
||||
image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15);
|
||||
return image;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+2
@@ -19,6 +19,8 @@ class SnapshotMonitor extends BaseBatch {
|
||||
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 房金部自动检测机器人
|
||||
// https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SnapshotMonitor(), args)
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package com.fangjia.script.temp.shenjian.address
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
|
||||
final DBCollection townConn = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址(估家科技)V2.xlsx", {
|
||||
List item ->
|
||||
def id = item['A']
|
||||
def province = item['D']
|
||||
def city = item['E']
|
||||
def area = item['F']
|
||||
String township = item['G']
|
||||
String community = item['H']
|
||||
if (StringUtils.isBlank(city)) {
|
||||
return ;
|
||||
}
|
||||
if (item['T'] != null) {
|
||||
return ;
|
||||
}
|
||||
// if (community == null) {
|
||||
// return ;
|
||||
// }
|
||||
if (township) {
|
||||
if (township.endsWith("镇") || township.endsWith("乡")) {
|
||||
item['T'] = 2
|
||||
} else if (township.endsWith("街道")) {
|
||||
item['T'] = 1
|
||||
}
|
||||
}
|
||||
// community = community.replaceAll("居委会", "")
|
||||
// community = community.replaceAll("居民委员会", "")
|
||||
// community = community.replaceAll("村民委员会", "")
|
||||
// community = community.replaceAll("村委会", "")
|
||||
//
|
||||
// if (ProvinceData.isGovernmentCity(city)) {
|
||||
// townConn.find(new BasicDBObject("province", "重庆市")
|
||||
// .append("region", area + "区")
|
||||
// .append("street", township)
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// if (item['T'] == null) {
|
||||
// townConn.find(new BasicDBObject("province", "重庆市")
|
||||
// .append("region", area + "区")
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// townConn.find(new BasicDBObject("city", city + "市")
|
||||
// .append("region", area + "区")
|
||||
// .append("street", township)
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// if (item['T'] == null) {
|
||||
// townConn.find(new BasicDBObject("city", city + "市")
|
||||
// .append("region", area + "区")
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
})
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.fangjia.script.temp.shenjian.address
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
Map<String, List<String>> map = new HashMap<>()
|
||||
List<List<String>> readList = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\(曹合1 )重庆农商行测试地址V8-1(街道、社区、街道代码补充)处理--新-芝.xlsx")
|
||||
for (List<String> result : readList) {
|
||||
map.put(result.get(0), result)
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", {
|
||||
List item ->
|
||||
def id = item['A']
|
||||
def township = item['G']
|
||||
def community = item['H']
|
||||
if (org.apache.commons.lang.StringUtils.isBlank(township)) {
|
||||
item['G'] = map[id]['F']
|
||||
item['Y'] = '人工乡镇'
|
||||
}
|
||||
if (org.apache.commons.lang.StringUtils.isBlank(community)) {
|
||||
item['H'] = map[id]['G']
|
||||
item['Z'] = '人工社区'
|
||||
}
|
||||
})
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.fangjia.script.temp.shenjian.address
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 补充详细地址中缺失的乡镇街道与居委社区
|
||||
*/
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", {
|
||||
List item ->
|
||||
def area = item['F']
|
||||
def township = item['G']
|
||||
def community = item['H']
|
||||
String detailAddr = item['I']
|
||||
|
||||
|
||||
if (detailAddr == '重庆市云阳县双江街道恒生场2-02') {
|
||||
print("test")
|
||||
}
|
||||
// 不包含街道与居委
|
||||
if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) &&
|
||||
!detailAddr.contains(township) && !detailAddr.contains(community) ) {
|
||||
if (StringUtils.isNotBlank(area)) {
|
||||
if (detailAddr.indexOf(area + "区") > 0) {
|
||||
int index = detailAddr.indexOf(area + "区") + area.length() + 1
|
||||
item['K'] = detailAddr.substring(0, index) + township + community +
|
||||
detailAddr.substring(index)
|
||||
} else if (detailAddr.indexOf(area + "县") > 0) {
|
||||
int index = detailAddr.indexOf(area + "县") + area.length() + 1
|
||||
item['K'] = detailAddr.substring(0, index) + township + community +
|
||||
detailAddr.substring(index)
|
||||
}
|
||||
}
|
||||
// 不包含社区
|
||||
} else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) &&
|
||||
detailAddr.contains(township) && !detailAddr.contains(community)) {
|
||||
if (detailAddr.indexOf(township) > 0) {
|
||||
int index = detailAddr.indexOf(township) + township.length()
|
||||
item['K'] = detailAddr.substring(0, index) + community +
|
||||
detailAddr.substring(index)
|
||||
}
|
||||
} // 只不包含街道
|
||||
else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) &&
|
||||
!detailAddr.contains(township) && detailAddr.contains(community)) {
|
||||
if (StringUtils.isNotBlank(area)) {
|
||||
if (detailAddr.indexOf(area + "区") > 0) {
|
||||
int index = detailAddr.indexOf(area + "区") + area.length() + 1
|
||||
item['K'] = detailAddr.substring(0, index) + township +
|
||||
detailAddr.substring(index)
|
||||
} else if (detailAddr.indexOf(area + "县") > 0) {
|
||||
int index = detailAddr.indexOf(area + "县") + area.length() + 1
|
||||
item['K'] = detailAddr.substring(0, index) + township +
|
||||
detailAddr.substring(index)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.temp.shenjian.address
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
//ExcelUtils.removeEmptyRow("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V10(优化补充社区冗余信息).xlsx")
|
||||
ners = ["houseno", "cellno", "floorno", "roomno"]
|
||||
Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组")
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item->
|
||||
|
||||
if (StringUtils.isNotBlank(item["K"])) {
|
||||
item["I"] = item["K"]
|
||||
}
|
||||
// def addr = item["B"].toString()
|
||||
// def detailAddr = item["I"]?.toString()
|
||||
// def pd = item["C"]
|
||||
// item['J'] = ""
|
||||
// if (StringUtils.isNotBlank(pd)) {
|
||||
// int len = addr.lastIndexOf(pd)
|
||||
// if (len > 3) {
|
||||
// String aim = addr.substring(len - 3, len)
|
||||
// int aimIndex = detailAddr.lastIndexOf(aim)
|
||||
// if (aimIndex > 0) {
|
||||
// item['J'] = detailAddr.substring(0, aimIndex + 3) + pd
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(detailAddr{
|
||||
// String group = DataUtil.match(pd, GROUP_PATTERN)
|
||||
// if(group == null ){
|
||||
// return
|
||||
// }
|
||||
// def arabic_group = null
|
||||
// if(!detailAddr.contains(group)){
|
||||
// arabic_group = CNNMFilter.qCNNumericToArabic(group)
|
||||
// }
|
||||
// if(group){
|
||||
// def flg = addr.substring(addr.indexOf(group) - 3, addr.indexOf(group))
|
||||
// flg = flg.replace("-", "")
|
||||
// if(arabic_group != null){
|
||||
// group = arabic_group
|
||||
// }
|
||||
// if(!detailAddr.contains(flg + group)){
|
||||
// println(item)
|
||||
// detailAddr = detailAddr.replace(group, '')
|
||||
// def idx = detailAddr.lastIndexOf(flg)
|
||||
// try{
|
||||
// detailAddr = detailAddr.substring(0, idx) + flg + group + detailAddr.substring(idx + flg.length())
|
||||
// item["j"] = detailAddr
|
||||
// }catch(Exception e){
|
||||
// item["W"] ="组替换失败"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// JSONObject result = request.getJsonContent("http://192.168.0.19:31080/FangJiaNlpService/ner",["address":addr], "utf-8")
|
||||
// def index = Integer.MAX_VALUE
|
||||
// for(JSONObject entity : result.entities){
|
||||
// if(entity.type in ners){
|
||||
// if(entity.start < index){
|
||||
// index = entity.start
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(index != Integer.MAX_VALUE){
|
||||
// item["C"] = addr.substring(index)
|
||||
// println(item)
|
||||
// }
|
||||
})
|
||||
+55
-69
@@ -10,6 +10,8 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,11 +29,13 @@ class SeaweedExport {
|
||||
|
||||
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||
|
||||
static def map = new ConcurrentHashMap()
|
||||
|
||||
static void main(String[] args) {
|
||||
def resultMap = new ConcurrentHashMap()
|
||||
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
|
||||
List<List<Object>> data = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\重庆样本匹配结果V4.xlsx")
|
||||
CountDownLatch latch = new CountDownLatch(data.size())
|
||||
for (List<Object> districtT : data) {
|
||||
final List<Object> district = districtT
|
||||
@@ -39,72 +43,55 @@ class SeaweedExport {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
def city = district.get(1)
|
||||
def district_code = district.get(2)
|
||||
def name = district.get(3)
|
||||
def matchedCity = district.get(22)
|
||||
def matchedRegion = district.get(23)
|
||||
def matchedName = district.get(24)
|
||||
def region;
|
||||
def no = district.get(0)
|
||||
def id = district.get(2)
|
||||
def ret = []
|
||||
if (id != null) {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("_id", new ObjectId(id))
|
||||
//query.put("region", matchedRegion)
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
//query.put("name", matchedName)
|
||||
|
||||
if (map.get("district_code") != null) {
|
||||
region = map.get("district_code")
|
||||
} else {
|
||||
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
|
||||
region = obj.fangjia_district
|
||||
map.put("district_code", region)
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("_id", 1)
|
||||
filter.put("developer", 1)
|
||||
filter.put("estate_company", 1)
|
||||
filter.put("cubage_percent", 1) // 容积率
|
||||
filter.put("virescence_percent", 1) // 绿化率
|
||||
filter.put("alias", 1)
|
||||
filter.put("car_site_count", 1)
|
||||
filter.put("all_area", 1)
|
||||
filter.put("loudong_count", 1)
|
||||
filter.put("household_count", 1)
|
||||
filter.put("estate_charge", 1)
|
||||
filter.put("open_date", 1) // 开盘时间
|
||||
filter.put("live_date", 1) // 交房时间
|
||||
filter.put("elevator", 1) // 电梯
|
||||
|
||||
DBObject obj = seaweed.findOne(new ObjectId(id))
|
||||
if (obj) {
|
||||
ret[0] = obj.estate_company
|
||||
ret[1] = obj.developer
|
||||
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
|
||||
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
|
||||
ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias)
|
||||
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
|
||||
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
|
||||
ret[7] = ""
|
||||
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
|
||||
ret[9] = obj.loudong_count
|
||||
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
|
||||
ret[11] = ""
|
||||
ret[12] = obj.estate_charge
|
||||
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
|
||||
ret[14] = obj.live_date
|
||||
ret[15] = obj.is_elevator == true ? "有" : (obj.is_elevator == false ? "无" : "")
|
||||
ret[16] = no
|
||||
resultMap.put(no, ret)
|
||||
}
|
||||
}
|
||||
if (matchedRegion == null) {
|
||||
matchedRegion = region
|
||||
matchedName = name
|
||||
}
|
||||
def ret = []
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city", city)
|
||||
query.put("region", matchedRegion)
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("name", matchedName)
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("_id", 1)
|
||||
filter.put("developer", 1)
|
||||
filter.put("estate_company", 1)
|
||||
filter.put("cubage_percent", 1) // 容积率
|
||||
filter.put("virescence_percent", 1) // 绿化率
|
||||
filter.put("alias", 1)
|
||||
filter.put("car_site_count", 1)
|
||||
filter.put("all_area", 1)
|
||||
filter.put("loudong_count", 1)
|
||||
filter.put("household_count", 1)
|
||||
filter.put("estate_charge", 1)
|
||||
filter.put("open_date", 1) // 开盘时间
|
||||
filter.put("live_date", 1) // 交房时间
|
||||
filter.put("lng", 1)
|
||||
filter.put("lat", 1)
|
||||
|
||||
DbUtils.seaweed.find(query).limit(1).each { obj ->
|
||||
ret[0] = obj.estate_company
|
||||
ret[1] = obj.developer
|
||||
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
|
||||
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
|
||||
ret[4] = SeaweedUtils.formatAlias(obj.alias)
|
||||
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
|
||||
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
|
||||
ret[7] = ""
|
||||
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
|
||||
ret[9] = obj.loudong_count
|
||||
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
|
||||
ret[11] = ""
|
||||
ret[12] = obj.estate_charge
|
||||
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
|
||||
ret[14] = obj.live_date
|
||||
ret[15] = obj.lng
|
||||
ret[16] = obj.lat
|
||||
}
|
||||
ret[17] = name
|
||||
resultMap.put(district_code + name, ret)
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
@@ -115,13 +102,12 @@ class SeaweedExport {
|
||||
};
|
||||
latch.await();
|
||||
def ret = [];
|
||||
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"];
|
||||
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "是否有电梯", "序号"];
|
||||
for (List<Object> district : data) {
|
||||
def district_code = district.get(2)
|
||||
def name = district.get(3)
|
||||
ret << resultMap.get(district_code + name)
|
||||
def no = district.get(0)
|
||||
ret << resultMap.get(no)
|
||||
};
|
||||
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
|
||||
com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
|
||||
executorService.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils
|
||||
class DingTalkUtils {
|
||||
|
||||
static main(String[] args) {
|
||||
Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ;
|
||||
// 要导出的月份第一天
|
||||
Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ;
|
||||
|
||||
String startTime = DateUtils.addMonth(now, -1) + " 00:00:00"
|
||||
String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss")
|
||||
|
||||
+2
-2
@@ -21,10 +21,10 @@ import java.util.concurrent.Executors
|
||||
class CommitteeLatLng {
|
||||
|
||||
static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(20)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
|
||||
static void main(String[] args) {
|
||||
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("name", "席庄二村委会"))
|
||||
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021))
|
||||
//dealLocation()
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user