Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -13,7 +13,7 @@ import java.util.*;
|
||||
*/
|
||||
public class BaiDuApiUtils {
|
||||
|
||||
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
|
||||
private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX";
|
||||
public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
|
||||
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
|
||||
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
|
||||
|
||||
@@ -1435,6 +1435,25 @@ public class DataUtil {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
public static List<String> matchAll(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null;
|
||||
}
|
||||
List list = new ArrayList();
|
||||
Matcher matcher = pattern.matcher(str);
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static boolean isNumeric(String str) {
|
||||
try {
|
||||
Double.parseDouble(str);
|
||||
|
||||
@@ -58,7 +58,7 @@ public class ExcelUtil {
|
||||
row.setHeightInPoints(35);
|
||||
Font font = workBook.createFont();
|
||||
font.setFontName("宋体");
|
||||
font.setBoldweight(Font.BOLDWEIGHT_BOLD);
|
||||
// font.setBoldweight(Font.BOLDWEIGHT_BOLD);
|
||||
font.setFontHeightInPoints((short) 12);
|
||||
CellStyle cellStyle = workBook.createCellStyle();
|
||||
cellStyle.setFont(font);
|
||||
|
||||
@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
|
||||
|
||||
public static void main(String[] args) {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
System.out.println(random(0, 10));
|
||||
System.out.println(random(-9, 9));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,8 @@ public class FjtEvaluation implements SqlEntity{
|
||||
|
||||
private String finishDateStr;//竣工时间
|
||||
private String useType;//房屋用途
|
||||
|
||||
private Date downPdfDate;//下载报告时间
|
||||
|
||||
@GeneratedValue(strategy = GenerationType.AUTO)
|
||||
@Id
|
||||
@@ -750,6 +752,16 @@ public class FjtEvaluation implements SqlEntity{
|
||||
this.useType = useType;
|
||||
}
|
||||
|
||||
|
||||
@Column(name = "down_pdf_date")
|
||||
public Date getDownPdfDate() {
|
||||
return downPdfDate;
|
||||
}
|
||||
|
||||
public void setDownPdfDate(Date downPdfDate) {
|
||||
this.downPdfDate = downPdfDate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FjtEvaluation{" +
|
||||
|
||||
@@ -38,6 +38,7 @@ public class OpenConfig extends Configuration {
|
||||
|
||||
private final String ipdatfile;
|
||||
private final String baiduAk;
|
||||
private final String defaultCity;
|
||||
|
||||
private final boolean isOpenAllAuthority;
|
||||
|
||||
@@ -54,6 +55,7 @@ public class OpenConfig extends Configuration {
|
||||
baiduAk = configuration.get("fangjia.baidu.ak");
|
||||
isOpenAllAuthority = configuration.getBoolean("fangjia.isOpenAllAuthority", false);
|
||||
enabledAddressMatch = configuration.getBoolean("fangjia.addressMatch.enabled",false);
|
||||
defaultCity = configuration.get("fangjia.default.city");
|
||||
}
|
||||
|
||||
public static OpenConfig getConfig() {
|
||||
@@ -134,4 +136,7 @@ public class OpenConfig extends Configuration {
|
||||
return enabledAddressMatch;
|
||||
}
|
||||
|
||||
public String getDefaultCity() {
|
||||
return defaultCity;
|
||||
}
|
||||
}
|
||||
|
||||
+226
@@ -89,6 +89,7 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService;
|
||||
import com.fangjia.service.open.WaitCrawlKeywordsService;
|
||||
import com.fangjia.service.sql.FjtEvaluationService;
|
||||
import com.fangjia.service.web.AddressService;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
@Authority(role = Role.USER)
|
||||
public class AddressController {
|
||||
@@ -106,6 +107,7 @@ public class AddressController {
|
||||
private HouseLoudongService houseLoudongService;
|
||||
private FjtEvaluationService fjtEvaluationService;
|
||||
private AddressMatchProxyService addressMatchProxyService;
|
||||
|
||||
private final static String AK = OpenConfig.getConfig().getBaiduAk();
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(AddressController.class);
|
||||
|
||||
@@ -117,6 +119,8 @@ public class AddressController {
|
||||
private static final String GRADE_NATION = "全国评分";
|
||||
private static final String GRADE_CONSUME = "消费评分";
|
||||
|
||||
public final static OpenConfig CONFIG = OpenConfig.getConfig();
|
||||
|
||||
/**
|
||||
* @deprecated see parse2Action
|
||||
* @param model
|
||||
@@ -1705,6 +1709,228 @@ public class AddressController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 地址标准化接口V2
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object standard2Action(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 五级地址标准化接口
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object fiveDivisionStructureAction(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 五级行政区划V2地址标准化接口
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object fiveDivisionStructure2Action(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 详细地址标准化接口
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object divisionStructureAction(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 地址标准化专用测试接口
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object structureAndCompleteAction(AddesssMatchModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
// 地址验证无效也输出标准化结果
|
||||
params.put("flag", "true");
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if(json.containsKey("result")){
|
||||
basicModel.setResult(json.get("result"));
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object similarityAction(AddesssMatchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
|
||||
|
||||
+1
-1
@@ -2363,7 +2363,7 @@ public class PropertyController {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("size(面积)不能为空"));
|
||||
return mav;
|
||||
}
|
||||
District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(),
|
||||
District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName().replace(" ",""),
|
||||
"district");
|
||||
if (null == district) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
|
||||
+1
-1
@@ -80,7 +80,7 @@ public class DistrictController {
|
||||
// response.sendRedirect("/coming");
|
||||
// return null;
|
||||
// }
|
||||
if("b".equals(login_user_type)){
|
||||
if("b".equals(login_user_type) || "gb".equals(login_user_type)){
|
||||
CacheUtils.putValue(CachePolicyEnum.LONG, UsersInfoService.class, weiXinId + "_city_"+ login_user_type , model.getCityName());
|
||||
}
|
||||
Long userId = Long.parseLong(CookieUtils.getCookieValue(request,"login_user_id"));
|
||||
|
||||
+53
-11
@@ -80,7 +80,19 @@ public class UserController {
|
||||
} else {
|
||||
weiXinId = CookieUtils.getCookieValue(request,"we_chat_open_id");
|
||||
}
|
||||
|
||||
String cacheWeiXinId = CookieUtils.getCookieValue(request,"login_enterpriseuser_weixinid");
|
||||
try {
|
||||
if(StringUtils.isNotBlank(weiXinId) && StringUtils.isNotBlank(cacheWeiXinId)){
|
||||
System.out.println("cookie weiXinId:"+weiXinId);
|
||||
response.sendRedirect("reqCityChoicePage?type=b&code="+model.getCode());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
if(weiXinId == null && StringUtils.isNotBlank(model.getCode())) {//微信公众号渤海银行
|
||||
System.out.println("bh OPEN_ID Ëæ»úÉú³É£º" + weiXinId);
|
||||
weiXinId = DataConvertUtil.getRandomString2(16);
|
||||
}
|
||||
if(weiXinId != null && !weiXinId.equals("")){
|
||||
@@ -115,33 +127,49 @@ public class UserController {
|
||||
if(weiXinId != null && !weiXinId.equals("")){
|
||||
CookieUtils.setCookie("we_chat_open_id", weiXinId, "/", 60 * 60 * 24 *7, response);
|
||||
}
|
||||
System.out.println("΢ÐÅOPEN_ID£º" + weiXinId);
|
||||
System.out.println("reqCityChoicePage ΢ÐÅOPEN_ID£º" + weiXinId);
|
||||
//企业用户入口查询企业账户开通的城市
|
||||
if(type.equals("b")){
|
||||
if(type.equals("b") || type.equals("gb")){
|
||||
//Boolean isLogin = usersInfoService.checkEnterpriseWeiXinUserIsLogin(weiXinId);
|
||||
String cacheWeiXinId = CookieUtils.getCookieValue(request,"login_enterpriseuser_weixinid");
|
||||
if(cacheWeiXinId != null) {
|
||||
Long userId = Long.parseLong(CookieUtils.getCookieValue(request,"login_user_id"));
|
||||
String loginUserIdstr = CookieUtils.getCookieValue(request,"login_user_id");
|
||||
if(cacheWeiXinId != null && StringUtils.isNotBlank(loginUserIdstr)) {
|
||||
Long userId = Long.parseLong(loginUserIdstr);
|
||||
CookieUtils.setCookie("login_user_id", userId + "", "/", 60 * 60 * 24 *7, response);
|
||||
try {
|
||||
FjtUsers users = fjtUsersService.findById(userId);
|
||||
List<String> citys = userCityService.findSupportCityByUserId(userId);
|
||||
if(citys == null || citys.isEmpty()){
|
||||
citys = Arrays.asList("ÉϺ£");
|
||||
}
|
||||
|
||||
if(users.getRole()==17 && type.equals("gb")){
|
||||
CookieUtils.removeCookie(request, response, "login_enterpriseuser_weixinid");
|
||||
response.sendRedirect("reqLoginPage");
|
||||
}else if(users.getRole()==12 && type.equals("b")){
|
||||
CookieUtils.removeCookie(request, response, "login_enterpriseuser_weixinid");
|
||||
response.sendRedirect("bhydLoginPage?code="+model.getCode());
|
||||
}else{
|
||||
List<String> citys = userCityService.findSupportCityByUserId(userId);
|
||||
if(citys == null || citys.isEmpty()){
|
||||
citys = Arrays.asList("ÉϺ£");
|
||||
}
|
||||
// if(users!=null && users.getRole() == 12){
|
||||
// citys = Arrays.asList("上海");
|
||||
// }else{
|
||||
// citys = userCityService.findSupportCityByUserId(userId);
|
||||
// }
|
||||
model.setCitys(citys);
|
||||
model.setCitys(citys);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
model.setCitys(new ArrayList<String>());
|
||||
}
|
||||
|
||||
} else {
|
||||
CookieUtils.removeCookie(request, response, "login_enterpriseuser_weixinid");
|
||||
try {
|
||||
response.sendRedirect("reqLoginPage");
|
||||
// if(type.equals("bh")){
|
||||
// response.sendRedirect("bhydLoginPage?code="+model.getCode());
|
||||
// }else{
|
||||
response.sendRedirect("reqLoginPage");
|
||||
// }
|
||||
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
@@ -173,6 +201,20 @@ public class UserController {
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
public Object bhloginOutAction(UserModel model,HttpServletRequest request, HttpServletResponse response, String city) {
|
||||
try {
|
||||
CookieUtils.removeCookie(request, response,"login_user_id");
|
||||
CookieUtils.removeCookie(request, response,"login_user_type");
|
||||
CookieUtils.removeCookie(request, response,"login_enterpriseuser_weixinid");
|
||||
response.sendRedirect("bhydLoginPage?code=bhydcode");
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public Object aboutAction(UserModel model) {
|
||||
return model;
|
||||
@@ -285,7 +327,7 @@ public class UserController {
|
||||
return resultModel;
|
||||
}
|
||||
CookieUtils.setCookie("login_user_id", user.getId() + "", "/", 60 * 60 * 24 *7, response);
|
||||
CookieUtils.setCookie("login_user_type", "b", "/", 60 * 60 * 24 *7, response);
|
||||
CookieUtils.setCookie("login_user_type", "gb", "/", 60 * 60 * 24 *7, response);
|
||||
CookieUtils.setCookie("login_enterpriseuser_weixinid", weiXinId, "/", 60 * 60 * 24 *7, response);
|
||||
//CacheUtils.putValue(CachePolicyEnum.LONG, UsersInfoService.class, weiXinId + "_" + Message.SMS_CODE_TYPE_ENTER_LOGIN , user.getId());
|
||||
resultModel.setResult(Message.RESULT_SUCCESS);
|
||||
|
||||
+12
-3
@@ -454,14 +454,22 @@ public class WeiXinEvaluationController {
|
||||
System.out.println("²³º£µØÖ·ËÑË÷ "+evaluation.getAddress());
|
||||
List<JSONObject> bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getAddress());
|
||||
JSONObject o = null;
|
||||
if(bhDists.size()<1){
|
||||
if(bhDists.size()==0){
|
||||
bhDists = BHBankUtils.getDistrictByBhbank(district.getName());
|
||||
}
|
||||
if(bhDists.size()==1){
|
||||
o = bhDists.get(0);
|
||||
}else if(bhDists.size()>1){
|
||||
int mod = Integer.MAX_VALUE;
|
||||
for(JSONObject jo : bhDists){
|
||||
double price = Double.valueOf((String)jo.get("price"));
|
||||
int bhprice = (int)price;
|
||||
int bhprice = 0;
|
||||
if (jo.get("price") == null || jo.get("price").equals("null")) {
|
||||
bhprice = 0;
|
||||
} else {
|
||||
double price = Double.valueOf((String)jo.get("price"));
|
||||
bhprice = (int)price;
|
||||
}
|
||||
|
||||
int bhmod = Math.abs(bhprice - avgPrice);
|
||||
if(bhmod<mod){
|
||||
mod = bhmod;
|
||||
@@ -1023,6 +1031,7 @@ public class WeiXinEvaluationController {
|
||||
|
||||
String reporNum = "147698"+ String.valueOf(evaluation.getId());
|
||||
evaluation.setReportNum(reporNum);
|
||||
evaluation.setDownPdfDate(new Date());
|
||||
fjtEvaluationService.updateEvaluation(evaluation);
|
||||
|
||||
try {
|
||||
|
||||
@@ -120,8 +120,19 @@ public class BHBankUtils {
|
||||
params.put("houseNo",ev.getRoomNum());
|
||||
}
|
||||
if(ev.getHouseArea()>0){
|
||||
params.put("groundArea", String.valueOf(ev.getHouseArea()));
|
||||
if(ev.getBasementArea()!=null && ev.getBasementArea()>0){
|
||||
double area = ev.getHouseArea() - ev.getBasementArea();
|
||||
if(area>0){
|
||||
params.put("groundArea", String.valueOf(area));
|
||||
}
|
||||
}else{
|
||||
params.put("groundArea", String.valueOf(ev.getHouseArea()));
|
||||
}
|
||||
}
|
||||
if(ev.getBasementArea()!=null && ev.getBasementArea()>0){
|
||||
params.put("undergroundArea", String.valueOf(ev.getBasementArea()));
|
||||
}
|
||||
|
||||
params.put("orientName", "ÄÏ");
|
||||
params.put("buildYear", ev.getFinishDateStr());
|
||||
fjwTotalPrice= (int)(fjwTotalPrice/10000) * 10000;
|
||||
@@ -430,7 +441,7 @@ public class BHBankUtils {
|
||||
Map params = new HashMap();
|
||||
params.put("queryId",queryId);
|
||||
if(StringUtils.isNotBlank(reportMsg)){
|
||||
params.put("reportMsg",reportMsg);
|
||||
params.put("reportMsg",reportMsg.replace(" ",""));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -100,4 +100,5 @@ fangjia.jms.email.queue=email.queue
|
||||
|
||||
fangjia.redis_nodes=192.168.0.79:6379,192.168.0.79:6380,192.168.0.79:6381,192.168.0.81:6382,192.168.0.81:6383,192.168.0.81:6384
|
||||
|
||||
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
|
||||
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
|
||||
fangjia.default.city = \u91CD\u5E86\u5E02
|
||||
@@ -1,8 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlrewrite>
|
||||
<global-params>
|
||||
<param value="st" name="sort" type="attach"/>
|
||||
<param value="pn" name="pager.currentPage" type="attach"/>
|
||||
<!-- <param value="st" name="sort" type="attach"/>-->
|
||||
<!-- <param value="pn" name="pager.currentPage" type="attach"/>-->
|
||||
</global-params>
|
||||
<rules>
|
||||
<rule controller="city" action="condition2" channel="v2">
|
||||
@@ -33,5 +33,15 @@
|
||||
<ladder>/property/avgPrice</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
<rule controller="address" action="standard2" channel="v2">
|
||||
<ladders>
|
||||
<ladder>/address/standard</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
<rule controller="address" action="fiveDivisionStructure2" channel="v2">
|
||||
<ladders>
|
||||
<ladder>/address/fiveDivisionStructure</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
</rules>
|
||||
</urlrewrite>
|
||||
@@ -26,7 +26,7 @@
|
||||
<div class="page page-current">
|
||||
<div class="content">
|
||||
<div class="top">
|
||||
<input id ="t" type="hidden" value="${type!'b'}">
|
||||
<input id ="t" type="hidden" value="${type!}">
|
||||
<#if !(cityName??) || cityName == "">
|
||||
<input id="cityName" value="上海" type="hidden">
|
||||
<#else>
|
||||
@@ -48,7 +48,7 @@
|
||||
<div class="list-block list-block01 media-list" id="searchSpan" style="margin-bottom:50px;">
|
||||
<ul></ul>
|
||||
</div>
|
||||
<#if type?? && type == 'b'>
|
||||
<#if type?? && (type == 'b' || type == 'gb')>
|
||||
<nav class="bar bar-tab">
|
||||
<#if role?? && role == 17>
|
||||
<a class="tab-item external" href="../weiXinEvaluation/bhhistory">估价记录</a>
|
||||
|
||||
@@ -152,7 +152,8 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#buildType").val("8")
|
||||
$("#houseFloor").show()
|
||||
$("#floorDesc").hide()
|
||||
$("#basementAreaLi").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -163,6 +164,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -173,6 +175,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -183,6 +186,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -193,6 +197,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -203,6 +208,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -213,6 +219,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").show()
|
||||
$("#basementAreaLi_intr").show()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -223,6 +230,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#floorDesc").show()
|
||||
// $("#houseFloor").hide()
|
||||
$("#basementAreaLi").hide()
|
||||
$("#basementAreaLi_intr").hide()
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -233,6 +241,7 @@ $(document).on('click', '.create-actions03', function () {
|
||||
$("#houseFloor").show()
|
||||
// $("#floorDesc").hide()
|
||||
$("#basementAreaLi").hide()
|
||||
$("#basementAreaLi_intr").hide()
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
</div>
|
||||
<div class="container">
|
||||
<div class="city">
|
||||
<#if type?? && type == 'b' && (citys??&&citys?size>0)>
|
||||
<#if type?? && (type == 'b' || type == 'gb') && (citys??&&citys?size>0)>
|
||||
<div class="city-list">
|
||||
<#list citys as bo>
|
||||
<p onclick="req2(this)">${bo}</p>
|
||||
@@ -54,7 +54,7 @@
|
||||
</#list>
|
||||
</div>
|
||||
</#if>
|
||||
<#if type?? && type == 'c' || type == 'b' && (citys??&&citys?size=0)>
|
||||
<#if type?? && type == 'c' || (type == 'b' || type == 'gb') && (citys??&&citys?size=0)>
|
||||
<div class="city-list curCity">
|
||||
<span class="city-letter" id="Cur">当前定位城市</span>
|
||||
</div>
|
||||
|
||||
@@ -84,7 +84,7 @@
|
||||
if(re.result == "fail"){
|
||||
$.toast(re.msg);
|
||||
} else {
|
||||
window.location.href = "reqCityChoicePage?type=b";
|
||||
window.location.href = "reqCityChoicePage?type=gb";
|
||||
}
|
||||
},error:function (data, status) {
|
||||
$.toast("发送失败,请重试");
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
|
||||
<nav class="bar bar-tab">
|
||||
<a class="tab-item external" href="../district/weXinList">返回首页</a>
|
||||
<a class="tab-item external" href="../user/bhydLoginPage">退出登录</a>
|
||||
<a class="tab-item external" href="../user/bhloginOut">退出登录</a>
|
||||
<#if role?? && role == 17>
|
||||
<a class="tab-item external" href="../user/setup">我的</a>
|
||||
</#if>
|
||||
|
||||
@@ -172,6 +172,13 @@
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li id="basementAreaLi_intr" style="color:#999;" class="dn">
|
||||
<div class="item-content">
|
||||
<div class="item-inner">
|
||||
*建筑面积中含有地下附属面积
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
<#if role?? && role==17 && loanInfoMap??>
|
||||
<p style="margin-top:-12px;">所属分类:${loanInfoMap.type!}类</p>
|
||||
<p style="margin-top:-10px;">可贷成数:${loanInfoMap.loan_ratio!}成</p>
|
||||
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(fjtEvaluation.totalPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
|
||||
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
|
||||
<p style="margin-top:-4px;">贷款利率:4.5%</p>
|
||||
</#if>
|
||||
<#else>
|
||||
@@ -88,7 +88,7 @@
|
||||
<div class="topic inputTopic01" id="inputTopic01a">
|
||||
<div class="topicTitle topicTitle01">
|
||||
<span>期望价:</span>
|
||||
<input class="input_price" type="textArea" name="dealPrice"> 万元
|
||||
<input class="input_price" type="textArea" name="dealPrice" <#if userTrades??>disabled</#if>> 万元
|
||||
<a class="submit_btn" id="submitbhPrice" style="border-left: none;">提交</a>
|
||||
<a class="submit_btn dn" id="refreshbhPrice" style="border-left: none;">刷新</a>
|
||||
</div>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.fangjia</groupId>
|
||||
<artifactId>fangjia-script</artifactId>
|
||||
@@ -62,7 +62,7 @@
|
||||
</resources>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<!--This plugin's configuration is used to store Eclipse m2e settings
|
||||
<!--This plugin's configuration is used to store Eclipse m2e settings
|
||||
only. It has no influence on the Maven build itself. -->
|
||||
<plugin>
|
||||
<groupId>org.eclipse.m2e</groupId>
|
||||
@@ -141,11 +141,11 @@
|
||||
<artifactId>activemq-pool</artifactId>
|
||||
<version>5.5.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>cn.emay</groupId>
|
||||
<artifactId>emayclient</artifactId>
|
||||
<version>4.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>cn.emay</groupId>
|
||||
<artifactId>emayclient</artifactId>
|
||||
<version>4.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.activemq</groupId>
|
||||
<artifactId>activemq-core</artifactId>
|
||||
@@ -366,18 +366,20 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi</artifactId>
|
||||
<!-- <version>3.17</version>-->
|
||||
<version>3.10-FINAL</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<!-- <version>3.17</version>-->
|
||||
<version>3.10-FINAL</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>xml-apis</groupId>
|
||||
<artifactId>xml-apis</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>xml-apis</groupId>
|
||||
<artifactId>xml-apis</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>
|
||||
@@ -476,18 +478,21 @@
|
||||
<artifactId>aliyun-java-sdk-address-purification</artifactId>
|
||||
<version>1.0.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.eclipsesource.j2v8</groupId>
|
||||
<artifactId>j2v8_win32_x86_64</artifactId>
|
||||
<version>4.6.0</version>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>easyexcel</artifactId>
|
||||
<version>2.2.8</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.18.20</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.msgpack</groupId>
|
||||
<artifactId>msgpack</artifactId>
|
||||
<version>0.6.12</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>elastic.co</id>
|
||||
<url>https://artifacts.elastic.co/maven</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
</project>
|
||||
|
||||
+13
-3
@@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -34,9 +35,12 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
if(buildingMap){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap)
|
||||
// rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat", headers, "utf-8", true);
|
||||
def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -97,7 +101,13 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
|
||||
+5
-1
@@ -44,6 +44,9 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
// if(["坑鹵","굇쑴","밤鹵","볼鹵","북례","�柏","柯소鏤","嶠볶","鮫갛","路헤"].contains(city)){
|
||||
// continue
|
||||
// }
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
|
||||
println city +" 小区量:"+dists.size()
|
||||
@@ -54,6 +57,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
@@ -168,7 +172,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "pa100"
|
||||
ret << "pa101"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().contains("弄")){
|
||||
|
||||
+121
-48
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
@@ -7,6 +8,7 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -19,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@@ -29,8 +32,8 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = 20210901//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
@@ -39,7 +42,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
@@ -50,12 +53,15 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
println citys.unique().size()
|
||||
|
||||
// for(def city : citys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// println citys.unique().size()
|
||||
def rows = []
|
||||
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
|
||||
@@ -76,12 +82,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
rows.addAll(rets)
|
||||
println city +" 成交共:"+rets.size()
|
||||
|
||||
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
@@ -92,10 +95,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
// def rets = []
|
||||
// rets << headers
|
||||
// rets.addAll(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
@@ -223,7 +223,6 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
@@ -231,11 +230,16 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
@@ -251,7 +255,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << 2
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
@@ -336,11 +340,17 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << trade.floor
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
@@ -356,7 +366,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
|
||||
ret << 2
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
@@ -390,18 +400,38 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getHouseRoom(def room, def hall, def toilet){
|
||||
def s = ""
|
||||
if(room){
|
||||
s = s + room +"室"
|
||||
def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){
|
||||
def housetype = null
|
||||
if(buildtype && ["0112","0110","0109"].contains(buildtype)){
|
||||
housetype = "07"
|
||||
}else if(buildtype && ["0107","0106","0105"].contains(buildtype)){
|
||||
housetype = "16"
|
||||
}else{
|
||||
if(roomcnt && roomcnt==4){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "14"
|
||||
}else{
|
||||
housetype = "11"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==3){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "13"
|
||||
}else{
|
||||
housetype = "10"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==2){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "12"
|
||||
}else{
|
||||
housetype = "09"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==1){
|
||||
housetype = "08"
|
||||
}else{
|
||||
housetype="99"
|
||||
}
|
||||
}
|
||||
if(hall){
|
||||
s = s + hall +"厅"
|
||||
}
|
||||
if(toilet){
|
||||
s = s + toilet +"卫"
|
||||
}
|
||||
return s;
|
||||
return housetype;
|
||||
}
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
@@ -413,15 +443,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
def startDate = null;
|
||||
// if(citys.contains(city)){
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }else{
|
||||
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -443,14 +467,19 @@ class BigdataDealBatch extends BaseBatch{
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// }
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
if(!trades && dist.address){
|
||||
def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)
|
||||
}
|
||||
|
||||
}
|
||||
// def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
// if(!trades && dist.address){
|
||||
// def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
// if(flag){
|
||||
// trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
|
||||
//
|
||||
// if(!trades){
|
||||
// trades = CcbCommonUtils.findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def trades = CcbCommonUtils.findTradesDetailById(dist, startDate, endDate)
|
||||
if(trades){
|
||||
// def unitPrice= ppeDistsMap.get(dist._id.toString())
|
||||
// if(unitPrice){
|
||||
@@ -476,11 +505,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def getYouDaData(statMap){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
// q.put("premisesName" , "呼玛二村")
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
@@ -540,4 +567,50 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for(def i=0;i<trades.size();i++){
|
||||
def t = trades[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}
|
||||
}
|
||||
println count_.get()+" "+t+" "+ rMap
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
}
|
||||
}
|
||||
|
||||
+165
-5
@@ -57,7 +57,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
@@ -74,8 +74,10 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211019, 20211118);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +98,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def commId = rMap.get("_id").toString()
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
// if(ret && ret.size()<100){
|
||||
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
|
||||
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
|
||||
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
// }
|
||||
if(ret){
|
||||
@@ -247,10 +250,167 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
return data
|
||||
}
|
||||
|
||||
def findHousesByDistId(def city, def commId, def cat, def startDate, def endDate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)).must(QueryBuilders.termQuery("match_seaweed_id", commId))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
if(seawater.name && seawater.name.contains("商住")){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type && seawater.estate_type.contains("商住")){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.purpose && seawater.purpose.contains("商住")){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
|
||||
println commId +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
|
||||
+140
-3
@@ -9,6 +9,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbCommonUtils {
|
||||
@@ -34,7 +35,7 @@ class CcbCommonUtils {
|
||||
|
||||
public static DSCollection dataStoreConnection = null;
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
public static def findCityCodeValue(){
|
||||
@@ -81,9 +82,10 @@ class CcbCommonUtils {
|
||||
return rets
|
||||
}
|
||||
|
||||
public static def findGuidePrice(city){
|
||||
public static def findGuidePrice(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
@@ -144,6 +146,38 @@ class CcbCommonUtils {
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetailById(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city", dist.city)
|
||||
// q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
q.put("match_id", dist._id.toString())
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
@@ -176,6 +210,38 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("address", dist.address)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
public static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
@@ -234,6 +300,26 @@ class CcbCommonUtils {
|
||||
return set
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictMapByOracle(def cityid){
|
||||
def sql = "select CITYID,COMMID,COMMNAME,DISTID from bigdata_community_gj where cityId='${cityid}' and status=0 ";
|
||||
def detailMap = [:]
|
||||
try{
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityid = r.CITYID
|
||||
map.commid = r.COMMID
|
||||
map.commname = r.COMMNAME
|
||||
map.distid = r.DISTID
|
||||
detailMap.put(r.COMMID, map)
|
||||
});
|
||||
}catch(Exception e){
|
||||
println "´íÎó£º"+e.getMessage()
|
||||
}
|
||||
|
||||
return detailMap
|
||||
}
|
||||
|
||||
public static def findAllDistrictsByOracle(def cityid){
|
||||
def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId='${cityid}' and status=0 ";
|
||||
def rows = []
|
||||
@@ -281,7 +367,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","江南人家")
|
||||
// q.put("name","ÌØ·¿½õÐåÒ»·½")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["ÆÕͨסլ","ÉÌס"]))
|
||||
|
||||
@@ -374,4 +460,55 @@ class CcbCommonUtils {
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static def findTradesByMonth(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', false))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
println city +" "+total
|
||||
for(def d : dbcursor){
|
||||
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id]
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
public static def updateDealPrice(id, match_city, match_region, match_name, match_id){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_city", match_city)
|
||||
udata.put("match_region", match_region)
|
||||
udata.put("match_name", match_name)
|
||||
udata.put("match_id", match_id)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
public static def findDistrictSeqno(){
|
||||
def sql = "select cityid,commid,seqno from bigdata_position_seqno";
|
||||
def map = new HashMap()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def detailMap = map.get(r.cityid)
|
||||
if(!detailMap){
|
||||
detailMap = [:]
|
||||
map.put(r.cityid, detailMap)
|
||||
}
|
||||
detailMap.put(r.commid, r.seqno)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
public static def findMaxSeqno(def cityid){
|
||||
def sql = "select max(seqno) as max_seqno from bigdata_position_seqno where cityid='${cityid}'"
|
||||
def maxSeqno = null;
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
maxSeqno = r.max_seqno
|
||||
});
|
||||
return maxSeqno
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jiangnanblank.SFTP
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class CcbFtpTestUtils {
|
||||
|
||||
static def ftpServer = "192.168.0.86"
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "ccb"
|
||||
static def ftpPassword = "ccb!@#"
|
||||
static def ftpPath = "/home/ccb/output/"
|
||||
|
||||
static def ftpPath_rh = "/home/ccb_rh/output/"
|
||||
|
||||
static SFTP sftp = new SFTP();
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
|
||||
|
||||
static def ftpUpload(type,outputPath_dist, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
if(type && type=="rh"){
|
||||
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
|
||||
}else{
|
||||
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static def delete(path){
|
||||
if(path.startsWith("/")){
|
||||
sftp.deleteSFTP(path, sftpChannel)
|
||||
}else{
|
||||
sftp.deleteFile(path)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def reConnect(){
|
||||
if(sftpChannel == null){
|
||||
println "sftpChannel 为空,需重连....."
|
||||
if(sftp == null){
|
||||
sftp = new SFTP();
|
||||
}
|
||||
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
}else{
|
||||
println "sftpChannel 正常....."
|
||||
}
|
||||
return sftpChannel;
|
||||
}
|
||||
|
||||
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
|
||||
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
|
||||
|
||||
def data = new FileInputStream(new File(dataFileName))
|
||||
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -19,13 +19,13 @@ class CcbFtpUtils {
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
|
||||
|
||||
static def ftpUpload(outputPath_dist, month){
|
||||
static def ftpUpload(type,outputPath_dist, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
if(month.toString().endsWith("28")){
|
||||
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
|
||||
}else{
|
||||
if(type && type=="rh"){
|
||||
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
|
||||
}else{
|
||||
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+401
@@ -0,0 +1,401 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 建行成交数据表优化
|
||||
*/
|
||||
class DealPriceDataProcess {
|
||||
|
||||
|
||||
|
||||
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()
|
||||
task.delDealByPe()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
def yearMonth = 202201
|
||||
def thirdMonth = 202112
|
||||
|
||||
// def rets = findAllDealByPe(yearMonth, 70)
|
||||
// for(def r : rets){
|
||||
// deleteDealPrice(r)
|
||||
// }
|
||||
def list = findAllThirdByPe(yearMonth, 70)
|
||||
for(def r : list){
|
||||
deletThirdPrice(r, thirdMonth)
|
||||
}
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
|
||||
for(def d : datas){
|
||||
// if(d['G']){
|
||||
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
|
||||
// for(def r : rets){
|
||||
// deleteDealPrice(r)
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(d['H']){
|
||||
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
|
||||
// for(def m : rets){
|
||||
// def precent = d['H']/100.0
|
||||
// def newPrice = null
|
||||
// if(m.unit_price > m.deal_price){
|
||||
// newPrice = m.deal_price*precent + m.deal_price
|
||||
// }else if(m.unit_price < m.deal_price){
|
||||
// newPrice = m.deal_price - (m.deal_price*precent)
|
||||
// }
|
||||
// if(newPrice){
|
||||
// updateDealPrice(m, newPrice.toInteger())
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(d['M']){
|
||||
// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
|
||||
// for(def r : rets){
|
||||
// deletThirdPrice(r, thirdMonth)
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void process() {
|
||||
def yearMonth = 202111
|
||||
def cityPeMap = getCityPPeIndex("D:\\数据\\建行\\房产估值\\成交对标标准.xlsx")
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def calcutePpeMap = findCaluateCityPPE(yearMonth)
|
||||
cityPeMap.each{c,v->
|
||||
// if(c=="呼和浩特"){
|
||||
def retMap = calcutePpeMap.get(c)
|
||||
if(retMap && retMap.ppe_val){
|
||||
def ppe = retMap.ppe_val
|
||||
if(ppe < v){
|
||||
def mod = v-ppe
|
||||
def count = retMap.case_count.toInteger() * mod/100.0
|
||||
if(count > count.toInteger()){
|
||||
count = count.toInteger() +1
|
||||
}
|
||||
processDealPrice(v, c, retMap.case_count, count.toInteger(), yearMonth)
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
def processDealPrice(def v, def city, def totalCount, def updateCount, def yearMonth){
|
||||
def rets = findAllGtePPeDatas(city, yearMonth)
|
||||
def commCount = findLtePPeCount(city,yearMonth)
|
||||
def count = 0
|
||||
for(def m : rets){
|
||||
if(m.ppe <= 20){
|
||||
def newPrice = null
|
||||
if(m.unit_price > m.deal_price){
|
||||
newPrice = m.deal_price*0.04 + m.deal_price
|
||||
}else if(m.unit_price < m.deal_price){
|
||||
newPrice = m.deal_price - (m.deal_price*0.04)
|
||||
}
|
||||
if(newPrice){
|
||||
def newPe = (m.unit_price-newPrice)*100/newPrice
|
||||
if(java.lang.Math.abs(newPe) <=15){
|
||||
count ++
|
||||
updateDealPrice(m, newPrice.toInteger())
|
||||
}
|
||||
}
|
||||
}else{
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def pre = (commCount + count)*100/(totalCount)
|
||||
println "修改价格:"+count+" ppe:"+pre
|
||||
if(pre < v){
|
||||
rets = rets.reverse()
|
||||
def delIndex = 0
|
||||
for(def r : rets){
|
||||
if(r.ppe > 25){
|
||||
deleteDealPrice(r)
|
||||
delIndex++
|
||||
pre = (commCount + count)*100/(totalCount -delIndex)
|
||||
println "删除成交:"+delIndex+" ppe:"+pre
|
||||
if(pre >= v){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def deleteDealPrice(r){
|
||||
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
|
||||
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def deletThirdPrice(r, thirdMonth){
|
||||
def sql = "delete from bigdata_third_party_price_gj where price_date=${thirdMonth} and commid='${r.commid}' and community_price=${r.target_avg_price}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def updateDealPrice(r, newPrice){
|
||||
def totalPrice = (newPrice * r.buildarea).toInteger()
|
||||
def sql = "update bigdata_deal_gj set DEALUNITPRICE=${newPrice},DEALTOTALPRICE=${totalPrice} where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def findLtePPeCount(city, year_month){
|
||||
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=${year_month} and cityname='${city}' and data_type='成交' and pe<=15";
|
||||
def count = 0
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findAllDealByPe(yearMonth, peVal){
|
||||
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityname = r.cityname
|
||||
map.cityid=r.cityid
|
||||
map.distid = r.distid
|
||||
map.commid = r.commid
|
||||
map.commname = r.commname
|
||||
map.deal_price = r.target_avg_price
|
||||
map.unit_price = r.unit_price
|
||||
map.buildarea = r.buildarea
|
||||
map.ppe = r.pe
|
||||
map.dealdate = r.dealdate
|
||||
rets << map
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
def findDealByPe(city, yearMonth, peVal){
|
||||
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}";
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityname = r.cityname
|
||||
map.cityid=r.cityid
|
||||
map.distid = r.distid
|
||||
map.commid = r.commid
|
||||
map.commname = r.commname
|
||||
map.deal_price = r.target_avg_price
|
||||
map.unit_price = r.unit_price
|
||||
map.buildarea = r.buildarea
|
||||
map.ppe = r.pe
|
||||
map.dealdate = r.dealdate
|
||||
rets << map
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findAllThirdByPe(yearMonth, peVal){
|
||||
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='第三方' and pe>${peVal}";
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.year_month = r.year_month
|
||||
map.commid = r.commid
|
||||
map.target_avg_price = r.target_avg_price
|
||||
rets << map
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
def findThirdByPe(city, yearMonth, peVal){
|
||||
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}";
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.year_month = r.year_month
|
||||
map.commid = r.commid
|
||||
map.target_avg_price = r.target_avg_price
|
||||
rets << map
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findAllGtePPeDatas(city, yearMonth){
|
||||
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc";
|
||||
def rets = []
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityname = r.cityname
|
||||
map.cityid=r.cityid
|
||||
map.distid = r.distid
|
||||
map.commid = r.commid
|
||||
map.commname = r.commname
|
||||
map.deal_price = r.target_avg_price
|
||||
map.unit_price = r.unit_price
|
||||
map.ppe = r.pe
|
||||
map.buildarea = r.buildarea
|
||||
map.dealdate = r.dealdate
|
||||
rets << map
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findCaluateCityPPE(def yearMonth){
|
||||
def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}";
|
||||
def map = [:]
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
def retMap = [:]
|
||||
retMap.put("ppe_val", r.DEAL_PPE15_COVERAGE)
|
||||
retMap.put("case_count", r.DEAL_TEST_CASE_NUM)
|
||||
map.put(r.CITYNAME, retMap)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def findCityByPpeRange(def cityname, def yearMonth, ppeStart, ppeEnd){
|
||||
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where cityname='${cityname}' and year_month=${yearMonth} and data_type='成交' and pe>${ppeStart} and pe<${ppeEnd}";
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityname = r.cityname
|
||||
map.cityid=r.cityid
|
||||
map.distid = r.distid
|
||||
map.commid = r.commid
|
||||
map.commname = r.commname
|
||||
map.deal_price = r.target_avg_price
|
||||
map.unit_price = r.unit_price
|
||||
map.ppe = r.pe
|
||||
map.buildarea = r.buildarea
|
||||
map.dealdate = r.dealdate
|
||||
rets << map
|
||||
});
|
||||
return rets
|
||||
}
|
||||
def getCityPPeIndex(def filePath){
|
||||
def datas = ExcelUtils.read(filePath,0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
if(d['A']){
|
||||
map.put(d['A'], d['B'].toString().toDouble().toInteger())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def addLianJiaPrice(){
|
||||
def citys = ["杭州","广州","东莞"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202110
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def cityid = cityCodeDataMap.get(city)
|
||||
def priceMap = findLianjiaAvgPriceCount(cityid, publishTime)
|
||||
def commids = findExistsThirdPartyPrice(cityid, month)
|
||||
|
||||
def distMap = CcbCommonUtils.findAllDistrictMapByOracle(cityCodeDataMap.get(city))
|
||||
|
||||
priceMap.each {commid, avg_price->
|
||||
if(!commids.contains(commid)){
|
||||
|
||||
def distDetailMap = distMap.get(commid)
|
||||
if(distDetailMap){
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << distDetailMap['cityid']
|
||||
ret << distDetailMap['distid']
|
||||
ret << commid
|
||||
ret << distDetailMap['commname']
|
||||
ret << "04"
|
||||
ret << "链家"
|
||||
ret << month
|
||||
ret << avg_price
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
rows << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ_2.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\TBIGDATA_THIRD_PARTY_PRICE_GJ_2.dat"
|
||||
}
|
||||
def header = []
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
def findExistsThirdPartyPrice(cityid, yearMonth){
|
||||
def sql = "select commid from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityid}' and price_date=${yearMonth} and data_source='链家'"
|
||||
def commids = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
commids << r.commid
|
||||
});
|
||||
return commids
|
||||
}
|
||||
|
||||
|
||||
def findLianjiaAvgPriceCount(cityid, publishTime){
|
||||
def sql = "select commid, avg(unitprice) as avg_price from bigdata_forsale_gj where cityid='${cityid}' and publishtime>=to_date('${publishTime}','yyyy-MM-dd') and datasource in ('1','4') group by commid";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
map.put(r.commid, r.avg_price.toInteger())
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = ["东莞"] //,"杭州""广州",
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202110
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202111)
|
||||
def total = 0
|
||||
for(def m : rets){
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
println city +" "+total
|
||||
}
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20"
|
||||
def rets = []
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.commid = r.commid
|
||||
map.price = r.target_avg_price
|
||||
rets << map
|
||||
});
|
||||
return rets
|
||||
}
|
||||
|
||||
}
|
||||
-148
@@ -1,148 +0,0 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
class DealPriceDataProcess {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new DealPriceDataProcess()
|
||||
task.process()
|
||||
}
|
||||
|
||||
void process() {
|
||||
def yearMonth = 202110
|
||||
def cityPeMap = getCityPPeIndex("D:\\数据\\建行\\房产估值\\成交对标标准.xlsx")
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def calcutePpeMap = findCaluateCityPPE(yearMonth)
|
||||
cityPeMap.each{c,v->
|
||||
if(c=="呼和浩特"){
|
||||
def retMap = calcutePpeMap.get(c)
|
||||
if(retMap && retMap.ppe_val){
|
||||
def ppe = retMap.ppe_val
|
||||
if(ppe < v){
|
||||
def mod = v-ppe
|
||||
def count = retMap.case_count.toInteger() * mod/100.0
|
||||
if(count > count.toInteger()){
|
||||
count = count.toInteger() +1
|
||||
}
|
||||
processDealPrice(v, c, retMap.case_count, count.toInteger(), yearMonth)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def processDealPrice(def v, def city, def totalCount, def updateCount, def yearMonth){
|
||||
def rets = findAllGtePPeDatas(city, yearMonth)
|
||||
def commCount = findLtePPeCount(city,yearMonth)
|
||||
def count = 0
|
||||
for(def m : rets){
|
||||
if(m.ppe <= 20){
|
||||
def newPrice = null
|
||||
if(m.unit_price > m.deal_price){
|
||||
newPrice = m.deal_price*0.04 + m.deal_price
|
||||
}else if(m.unit_price < m.deal_price){
|
||||
newPrice = m.deal_price - (m.deal_price*0.04)
|
||||
}
|
||||
if(newPrice){
|
||||
def newPe = (m.unit_price-newPrice)*100/newPrice
|
||||
if(java.lang.Math.abs(newPe) <=15){
|
||||
count ++
|
||||
updateDealPrice(m, newPrice.toInteger())
|
||||
}
|
||||
}
|
||||
}else{
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def pre = (commCount + count)*100/(totalCount)
|
||||
println "修改价格:"+count+" ppe:"+pre
|
||||
if(pre < v){
|
||||
rets = rets.reverse()
|
||||
def delIndex = 0
|
||||
for(def r : rets){
|
||||
if(r.ppe > 25){
|
||||
deleteDealPrice(r)
|
||||
delIndex++
|
||||
pre = (commCount + count)*100/(totalCount -delIndex)
|
||||
println "删除成交:"+delIndex+" ppe:"+pre
|
||||
if(pre >= v){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def deleteDealPrice(r){
|
||||
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
|
||||
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def updateDealPrice(r, newPrice){
|
||||
def totalPrice = (newPrice * r.buildarea).toInteger()
|
||||
def sql = "update bigdata_deal_gj set DEALUNITPRICE=${newPrice},DEALTOTALPRICE=${totalPrice} where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def findLtePPeCount(city, year_month){
|
||||
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=${year_month} and cityname='${city}' and data_type='成交' and pe<=15";
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findAllGtePPeDatas(city, yearMonth){
|
||||
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc";
|
||||
def rets = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityname = r.cityname
|
||||
map.cityid=r.cityid
|
||||
map.distid = r.distid
|
||||
map.commid = r.commid
|
||||
map.commname = r.commname
|
||||
map.deal_price = r.target_avg_price
|
||||
map.unit_price = r.unit_price
|
||||
map.ppe = r.pe
|
||||
map.buildarea = r.buildarea
|
||||
map.dealdate = r.dealdate
|
||||
rets << map
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findCaluateCityPPE(def yearMonth){
|
||||
def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def retMap = [:]
|
||||
retMap.put("ppe_val", r.DEAL_PPE15_COVERAGE)
|
||||
retMap.put("case_count", r.DEAL_TEST_CASE_NUM)
|
||||
map.put(r.CITYNAME, retMap)
|
||||
});
|
||||
return map
|
||||
}
|
||||
def getCityPPeIndex(def filePath){
|
||||
def datas = ExcelUtils.read(filePath,0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
if(d['A']){
|
||||
map.put(d['A'], d['B'].toString().toDouble().toInteger())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
}
|
||||
+178
-2
@@ -1,15 +1,20 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -37,15 +42,74 @@ class DistrictDelete {
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
// task.gzRun();
|
||||
// task.deleteDistrict()
|
||||
task.deleteCommPrice()
|
||||
// task.deleteCommPrice()
|
||||
// task.distHouseCountStat()
|
||||
task.addThirdPartyStat()
|
||||
// task.processRegion()
|
||||
}
|
||||
def processRegion(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
|
||||
if(d['B']){
|
||||
def region = RegionData.formatRegion("重庆",d['B'])
|
||||
if(region){
|
||||
d['B'] = region
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addThirdPartyStat(){
|
||||
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
|
||||
def idMaps = [:]
|
||||
for(def d : datas){
|
||||
def id = d['A']
|
||||
idMaps.put(id, d['D'])
|
||||
}
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
def totalCount = new AtomicLong()
|
||||
def map = findThirdPartyPrice("上海")
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def thirdLists = Collections.synchronizedList(new ArrayList());
|
||||
thirdLists << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = idMaps.get(d._id.toString())
|
||||
if (!count) {
|
||||
def dist = CommonDatautils.findDistrictById(d._id.toString())
|
||||
def priceMap = map.get(dist.region + "##" + dist.name)
|
||||
|
||||
if (priceMap) {
|
||||
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
|
||||
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
|
||||
}
|
||||
}
|
||||
// println totalCount.get() + " " + d.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -60,6 +124,40 @@ class DistrictDelete {
|
||||
}
|
||||
}
|
||||
|
||||
def distHouseCountStat(){
|
||||
def city = "重庆"
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def distCount = districts.size()
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
rets << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
|
||||
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
|
||||
if(count){
|
||||
rets << [dist._id.toString(), dist.region, dist.name, count]
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
|
||||
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
@@ -259,4 +357,82 @@ class DistrictDelete {
|
||||
sqlInstance.execute(sql)
|
||||
println "ɾ³ý£º"+tableName+" "+commid
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def prices = map.get(key)
|
||||
if(!prices){
|
||||
prices = []
|
||||
map.put(key, prices)
|
||||
}
|
||||
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
prices << db.zhugezhaofang_esf_price
|
||||
}
|
||||
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
prices << db.anjuke_esf_price
|
||||
}
|
||||
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
prices << db.get("58tc_esf_price")
|
||||
}
|
||||
if(db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
prices << db.soufang_esf_price
|
||||
}
|
||||
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
prices << db.lianjia_esf_price
|
||||
}
|
||||
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
prices << db.cityhouse_esf_price
|
||||
}
|
||||
|
||||
if(db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
prices << db.maitian_esf_price
|
||||
}
|
||||
|
||||
if(db.beike_esf_price && db.beike_esf_price>3000){
|
||||
prices << db.beike_esf_price
|
||||
}
|
||||
|
||||
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
prices << db.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
prices << db.zhongyuan_esf_price
|
||||
}
|
||||
|
||||
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
prices << db.tuboshi_esf_price
|
||||
}
|
||||
|
||||
if( db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
prices << db.qfang_esf_price
|
||||
}
|
||||
|
||||
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
prices << db.fangtan_esf_price
|
||||
}
|
||||
|
||||
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
prices << db.fangduoduo_esf_price
|
||||
}
|
||||
|
||||
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
prices << db.get("5i5j_esf_price")
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findGuidePrice(city)
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-03-1", "2022-04-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+262
-82
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
@@ -11,7 +12,7 @@ import java.sql.ResultSet
|
||||
|
||||
class ExportOracleData {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
@@ -20,28 +21,44 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
def run() {
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
// exportBigdataDistrictData()
|
||||
// exportBigdataCommunity()
|
||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// exportBigdataForsale("2021-10-19")
|
||||
// exportGuidedPrice("2021-10-01","2021-11-01")
|
||||
//不会变
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()
|
||||
exportBigdataDistrictData()
|
||||
|
||||
exportGuidedPrice()//随人工改变
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()
|
||||
exportCommunityStaging()
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-02-19")
|
||||
exportBigdataDeal("2022-02-01")
|
||||
|
||||
|
||||
// exportThirdPartyPrice()
|
||||
//计算结果相关
|
||||
// exportAvmEvaluationGJNB(curMonth)
|
||||
// exportAVMEvaluationItemGJNB(curMonth)
|
||||
//
|
||||
// exportCityPrice(curMonth)
|
||||
// exportRegionPrice(curMonth)
|
||||
//
|
||||
// exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
// exportCommunityBasePrice(curMonth)
|
||||
//
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
exportThirdPartyPrice2()
|
||||
// exportBigdataDeal()
|
||||
// exportCommunityGuidedPrice()
|
||||
// exportAvmEvaluationGJNB()
|
||||
// exportAVMEvaluationItemGJNB()
|
||||
// exportCommunityBasePrice()
|
||||
// exportCommunityBasePriceGj()
|
||||
// exportCityPrice()
|
||||
// exportRegionPrice()
|
||||
// def curDate = curMonth.toString()+"28"
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//ok文件
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
@@ -64,14 +81,14 @@ class ExportOracleData {
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -106,7 +123,7 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_POI_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_POI_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -133,25 +150,31 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_STAGING_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "COMMUNITY_STAGING_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBuileType() {
|
||||
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj";
|
||||
// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
// for(def city : citys){
|
||||
// def cityid = cityCodeDataMap.get(city)
|
||||
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj";
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
// }
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
@@ -161,7 +184,7 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -198,13 +221,13 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_DISTRICT_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_DISTRICT_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportGuidedPrice(def startDate, def endDate) {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=to_date('${startDate}','yyyy-MM-dd') and dealdate<to_date('${endDate}','yyyy-MM-dd')";
|
||||
def exportGuidedPrice() {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -225,20 +248,20 @@ class ExportOracleData {
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "GUIDED_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityGuidedPrice() {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj";
|
||||
def exportCommunityGuidedPrice(def date) {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -266,16 +289,16 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDeal() {
|
||||
def exportBigdataDeal(def dealDate) {
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
|
||||
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj";
|
||||
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -356,7 +379,7 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_DEAL_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -405,12 +428,13 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
|
||||
//CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportThirdPartyPrice2() {
|
||||
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from third_party_price";
|
||||
def exportThirdPartyPrice2(price_date) {
|
||||
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} ";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -429,17 +453,17 @@ class ExportOracleData {
|
||||
allRows << data
|
||||
})
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
|
||||
//CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
}
|
||||
|
||||
|
||||
@@ -484,7 +508,7 @@ class ExportOracleData {
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -493,7 +517,7 @@ class ExportOracleData {
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -503,7 +527,7 @@ class ExportOracleData {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -585,16 +609,16 @@ class ExportOracleData {
|
||||
|
||||
}
|
||||
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportAvmEvaluationGJNB(){
|
||||
def exportAvmEvaluationGJNB(yearMonth){
|
||||
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
|
||||
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
|
||||
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH >= 202106";
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -637,22 +661,26 @@ class ExportOracleData {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
|
||||
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
|
||||
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportAVMEvaluationItemGJNB(){
|
||||
def exportAVMEvaluationItemGJNB(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
|
||||
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH >= 202106";
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -685,18 +713,22 @@ class ExportOracleData {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj";
|
||||
def exportCommunityBasePrice(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -724,22 +756,61 @@ class ExportOracleData {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePriceGj(){
|
||||
def exportCommunityBasePriceGj(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCityPrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj";
|
||||
def exportCityPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -756,19 +827,19 @@ class ExportOracleData {
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "CITY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "CITY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportRegionPrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ";
|
||||
def exportRegionPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -787,24 +858,133 @@ class ExportOracleData {
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//
|
||||
// for(def city : citys) {
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.CITYID
|
||||
data << r.CITY_NAME
|
||||
data << r.DISTRICT_CODE
|
||||
data << r.DISTRICT_NAME
|
||||
data << r.COMMUNITY_NAME
|
||||
data << r.PQ_NAME
|
||||
data << r.BASE_PRICE
|
||||
data << r.CONFIDENCE_LEVEL
|
||||
data << r.CNT
|
||||
data << r.MIN_PRICE
|
||||
data << r.MEDIANPRICE
|
||||
data << r.MEANPRICE
|
||||
data << r.MAX_PRICE
|
||||
data << r.TIME
|
||||
data << r.FIRSTDATAT
|
||||
data << r.THIRDDATA
|
||||
data << r.RELATIVEERROR
|
||||
data << r.SKEWNESS
|
||||
data << r.PQ_PRICE_RANGE
|
||||
data << r.DISTRICT_PRICE_RANGE
|
||||
data << r.DESCRIPTION
|
||||
data << r.PRIV_BASE_PRICE
|
||||
data << r.IS_ARTTFICIAL
|
||||
data << r.PRIV_FINAL_PRICE
|
||||
data << r.CREATE_DATE
|
||||
data << r.PRICE
|
||||
data << r.PRICE_SOURCE
|
||||
data << r.CASE_MD_PRICE
|
||||
data << r.CASE_AVG_PRICE
|
||||
data << r.INDEX_LABEL
|
||||
data << r.MAIN_PROPERTY
|
||||
data << r.COMMTYPE
|
||||
data << r.NEW_DEAL_PRICE
|
||||
data << r.NEW_DEAL_DATE
|
||||
data << r.PRICE_MOM_RATIO
|
||||
data << r.MAX_MOM_RATIO
|
||||
data << r.PRICE_YOY_RATIO
|
||||
data << r.SATUR_RATIO
|
||||
data << r.NEIGHBOR_PRICE
|
||||
allRows << data
|
||||
});
|
||||
println allRows.size()
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
// }
|
||||
println "community_p_describe_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "community_p_describe_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommCaseDetailGj(month){
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.NETLISTING_ID
|
||||
data << r.PRICE_DATE
|
||||
data << r.CREATE_DATE
|
||||
allRows << data
|
||||
if(allRows.size()%200000==0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
allRows.clear()
|
||||
}
|
||||
});
|
||||
if(allRows.size()>0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
println "comm_case_detail_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "comm_case_detail_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportLastFile(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def dataCodeFileName = "/ccb/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, curDate)
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
|
||||
class ExportOracleEvaluationData {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportOracleEvaluationData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
//计算结果相关
|
||||
exportAvmEvaluationGJNB(curMonth)
|
||||
exportAVMEvaluationItemGJNB(curMonth)
|
||||
|
||||
exportCityPrice(curMonth)
|
||||
exportRegionPrice(curMonth)
|
||||
|
||||
exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
exportCommunityBasePrice(curMonth)
|
||||
|
||||
exportCommunityPdescribeGj(curMonth)
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// ok文件
|
||||
exportOkFile()
|
||||
}
|
||||
|
||||
def exportAvmEvaluationGJNB(yearMonth){
|
||||
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
|
||||
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
|
||||
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.COMMUNITY_NUM
|
||||
data << r.HAS_PRICE_COMMUNITY_NUM
|
||||
data << r.PRICE_COVERAGE
|
||||
data << r.ARTIFICIAL_CALC_NUM
|
||||
data << r.ARTIFICIAL_CALC_COVERAGE
|
||||
data << r.SAMPLE_CALC_NUM
|
||||
data << r.SAMPLE_CALC_COVERAGE
|
||||
data << r.ANN_CALC_NUM
|
||||
data << r.ANN_CALC_COVERAGE
|
||||
data << r.TEST_NUM
|
||||
data << r.TEST_COMM_NUM
|
||||
data << r.TEST_COMM_COVERAGE
|
||||
data << r.PPE15_COVERAGE
|
||||
data << r.RATE_MEDIAN
|
||||
data << r.COEFFICIENT
|
||||
data << r.DEAL_TEST_CASE_NUM
|
||||
data << r.DEAL_TEST_COMM_NUM
|
||||
data << r.DEAL_TEST_COMM_COVERAGE
|
||||
data << r.DEAL_PPE15_COVERAGE
|
||||
data << r.DEAL_COEFFICIENT
|
||||
data << r.DEAL_RATE_MEDIAN
|
||||
data << r.THIRD_TEST_CASE_NUM
|
||||
data << r.THIRD_TEST_COMM_NUM
|
||||
data << r.THIRD_TEST_COMM_COVERAGE
|
||||
data << r.THIRD_PPE15_COVERAGE
|
||||
data << r.THIRD_COEFFICIENT
|
||||
data << r.THIRD_RATE_MEDIAN
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
|
||||
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
|
||||
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportAVMEvaluationItemGJNB(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
|
||||
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.CONFIDENCE
|
||||
data << r.TARGET_AVG_PRICE
|
||||
data << r.DEALDATE
|
||||
data << r.UNIT_PRICE
|
||||
data << r.BUILDAREA
|
||||
data << r.FITMENT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.DATA_TYPE
|
||||
data << r.RATE
|
||||
data << r.PE
|
||||
data << r.CREATETIME
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePrice(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePriceGj(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCityPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "CITY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "CITY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportRegionPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//
|
||||
// for(def city : citys) {
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
|
||||
// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.CITYID
|
||||
data << r.CITY_NAME
|
||||
data << r.DISTRICT_CODE
|
||||
data << r.DISTRICT_NAME
|
||||
data << r.COMMUNITY_NAME
|
||||
data << r.PQ_NAME
|
||||
data << r.BASE_PRICE
|
||||
data << r.CONFIDENCE_LEVEL
|
||||
data << r.CNT
|
||||
data << r.MIN_PRICE
|
||||
data << r.MEDIANPRICE
|
||||
data << r.MEANPRICE
|
||||
data << r.MAX_PRICE
|
||||
data << r.TIME
|
||||
data << r.FIRSTDATAT
|
||||
data << r.THIRDDATA
|
||||
data << r.RELATIVEERROR
|
||||
data << r.SKEWNESS
|
||||
data << r.PQ_PRICE_RANGE
|
||||
data << r.DISTRICT_PRICE_RANGE
|
||||
data << r.DESCRIPTION
|
||||
data << r.PRIV_BASE_PRICE
|
||||
data << r.IS_ARTTFICIAL
|
||||
data << r.PRIV_FINAL_PRICE
|
||||
data << r.CREATE_DATE
|
||||
data << r.PRICE
|
||||
data << r.PRICE_SOURCE
|
||||
data << r.CASE_MD_PRICE
|
||||
data << r.CASE_AVG_PRICE
|
||||
data << r.INDEX_LABEL
|
||||
data << r.MAIN_PROPERTY
|
||||
data << r.COMMTYPE
|
||||
data << r.NEW_DEAL_PRICE
|
||||
data << r.NEW_DEAL_DATE
|
||||
data << r.PRICE_MOM_RATIO
|
||||
data << r.MAX_MOM_RATIO
|
||||
data << r.PRICE_YOY_RATIO
|
||||
data << r.SATUR_RATIO
|
||||
data << r.NEIGHBOR_PRICE
|
||||
allRows << data
|
||||
});
|
||||
println allRows.size()
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
// }
|
||||
println "community_p_describe_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "community_p_describe_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommCaseDetailGj(month){
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.NETLISTING_ID
|
||||
data << r.PRICE_DATE
|
||||
data << r.CREATE_DATE
|
||||
allRows << data
|
||||
if(allRows.size()%200000==0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
allRows.clear()
|
||||
}
|
||||
});
|
||||
if(allRows.size()>0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
println "comm_case_detail_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "comm_case_detail_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOkFile(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def dataCodeFileName = "/ccb/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
+11
-11
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
@@ -53,24 +53,24 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
//添加搜房的参考价
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
// def datas = findFangDatas(districts)
|
||||
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
|
||||
// rows.addAll(fangRets)
|
||||
|
||||
//添加保亭文昌的指导价
|
||||
def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
|
||||
def cityDataMap = [:]
|
||||
for(def d : cityDatas){
|
||||
cityDataMap.put(d['A'], d['E'])
|
||||
}
|
||||
def personDatas = findPersonDatas(districts, cityDataMap);
|
||||
def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
|
||||
rows.addAll(personRets)
|
||||
// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
|
||||
// def cityDataMap = [:]
|
||||
// for(def d : cityDatas){
|
||||
// cityDataMap.put(d['A'], d['E'])
|
||||
// }
|
||||
// def personDatas = findPersonDatas(districts, cityDataMap);
|
||||
// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
|
||||
// rows.addAll(personRets)
|
||||
|
||||
}
|
||||
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
// def rets = []
|
||||
|
||||
+32
-11
@@ -4,6 +4,8 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
|
||||
class LoadGuide {
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
@@ -14,13 +16,32 @@ class LoadGuide {
|
||||
}
|
||||
|
||||
def run(){
|
||||
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
|
||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
|
||||
|
||||
}
|
||||
def match(def filename){
|
||||
def city = "西安"
|
||||
ExcelUtils.modify(filename,{d->
|
||||
def newRegion = RegionData.formatRegion(city, d['A'])
|
||||
if(newRegion){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
|
||||
if(rMap){
|
||||
d['D'] = city
|
||||
d['E'] = rMap.region
|
||||
d['F'] = rMap.name
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
@@ -28,15 +49,15 @@ class LoadGuide {
|
||||
def datas = ExcelUtils.read(filePath,0, 1)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
if(d['D']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.city = d['D']
|
||||
o.region = d['A']
|
||||
o.name = d['C']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['D']
|
||||
o.match_region = d['E']
|
||||
o.match_name = d['F']
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
@@ -57,11 +78,11 @@ class LoadGuide {
|
||||
o.city = d['E']
|
||||
o.region = d['A']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.house_count = d['C'].toString().toDouble().toInteger()
|
||||
// o.house_count = d['C'].toString().toDouble().toInteger()
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
|
||||
+2
-2
@@ -36,11 +36,11 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
|
||||
|
||||
def results =[]
|
||||
def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
results << headers
|
||||
results.addAll(rows)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
|
||||
//ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
+242
-79
@@ -27,7 +27,13 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.*;
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
@@ -56,14 +62,12 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
// println city +" "+count
|
||||
//}
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx"
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx"
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
// dealMap.put(d['A'], d['B'])
|
||||
//}
|
||||
//
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// d['Q'] = dealMap.get(d['C'])
|
||||
@@ -97,7 +101,7 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
// if (city) {
|
||||
// def cityid = cityCodeDatas.get(city)
|
||||
// def total = findCount(cityid, sqlInstance)
|
||||
// d['H'] = total
|
||||
// d['P'] = total
|
||||
// }
|
||||
//},0)
|
||||
|
||||
@@ -220,15 +224,15 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
}
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\建行成交\\成交数据_1025 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance, "2021-06-01", "2021-07-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance, 202107)
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// def oldcount = dealMap.get(d['C'])
|
||||
// d['F'] = oldcount
|
||||
// d['V'] = oldcount
|
||||
// def casecount = testMap.get(d['C'])
|
||||
// d['G'] = casecount
|
||||
// d['W'] = casecount
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
@@ -243,7 +247,7 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
//},0)
|
||||
|
||||
//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\需要修正的一期数据.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(obj && obj.name.contains("一期")){
|
||||
@@ -266,81 +270,165 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d->
|
||||
if(d['A']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
|
||||
def alias = []
|
||||
if(d['H'] && d['H']!=dist.name){
|
||||
alias << d['H']
|
||||
if(d['H'].contains("·")){
|
||||
alias << d['H'].replace("·", "")
|
||||
}
|
||||
if(dist.alias){
|
||||
for(def aliasName : dist.alias){
|
||||
if(!alias.contains(aliasName)){
|
||||
alias << aliasName
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def address = null
|
||||
if(d['I']){
|
||||
address = AddressUtil.formatAddress(dist.city, d['I'])
|
||||
}
|
||||
|
||||
Long lng = null
|
||||
Long lat = null
|
||||
Long lat2 = null
|
||||
Long lng2 = null
|
||||
if(d['J']){
|
||||
lng = d['J'].toString().toDouble() * 10000000000l;
|
||||
lat = d['K'].toString().toDouble() * 10000000000l;
|
||||
def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
|
||||
if(json != null && json.containsKey("x")) {
|
||||
lat2 = json.get("y") * 10000000000l
|
||||
lng2 = json.get("x") * 10000000000l
|
||||
}
|
||||
}
|
||||
|
||||
def region = null
|
||||
if(dist.region!=d['F']){
|
||||
region = dist.region
|
||||
}
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
|
||||
def flag = d['M']
|
||||
if(flag==0.0){
|
||||
if(d['D']){
|
||||
def udata = new BasicDBObject()
|
||||
if(region){
|
||||
udata.put("region", region)
|
||||
}
|
||||
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
if(lat2){
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
}
|
||||
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
if(udata){
|
||||
index++
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println index+ " "+dist.city +" "+dist.name+" "+udata
|
||||
}
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['D']))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
index++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def citys = CcbCommonUtils.citys
|
||||
//def rets = []
|
||||
//for(def city : citys){
|
||||
// def count = CommonDatautils.findSeaweeds(city, "district", 1)
|
||||
// rets << [city, count]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据")
|
||||
|
||||
//def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"]
|
||||
//for(def city : citys){
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"])
|
||||
// def index = 0
|
||||
// for(def dist : districts){
|
||||
// if(dist.name_cat && dist.name_cat==1){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 0)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// }
|
||||
// }
|
||||
// println city +" "+index
|
||||
//}
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d->
|
||||
// if(d['C']){
|
||||
//// def udata = new BasicDBObject()
|
||||
//// udata.put("name_cat", 0)
|
||||
//// udata.put("m_date",new Date())
|
||||
////
|
||||
//// def q = new BasicDBObject()
|
||||
//// q.put("_id", new ObjectId(d['C']))
|
||||
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// def count = findDistCount(d['C'], sqlInstance);
|
||||
// if(count==0){
|
||||
// println d['C']
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
//
|
||||
// def alias = []
|
||||
// if(d['H'] && d['H']!=dist.name){
|
||||
// alias << d['H']
|
||||
// if(d['H'].contains("·")){
|
||||
// alias << d['H'].replace("·", "")
|
||||
// }
|
||||
// if(dist.alias){
|
||||
// for(def aliasName : dist.alias){
|
||||
// if(!alias.contains(aliasName)){
|
||||
// alias << aliasName
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def address = null
|
||||
// if(d['I']){
|
||||
// address = AddressUtil.formatAddress(dist.city, d['I'])
|
||||
// }
|
||||
//
|
||||
// Long lng = null
|
||||
// Long lat = null
|
||||
// Long lat2 = null
|
||||
// Long lng2 = null
|
||||
// if(d['J']){
|
||||
// lng = d['J'].toString().toDouble() * 10000000000l;
|
||||
// lat = d['K'].toString().toDouble() * 10000000000l;
|
||||
// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
|
||||
// if(json != null && json.containsKey("x")) {
|
||||
// lat2 = json.get("y") * 10000000000l
|
||||
// lng2 = json.get("x") * 10000000000l
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def region = null
|
||||
// if(dist.region!=d['F']){
|
||||
// region = dist.region
|
||||
// }
|
||||
//
|
||||
// def udata = new BasicDBObject()
|
||||
// if(region){
|
||||
// udata.put("region", region)
|
||||
// }
|
||||
//
|
||||
// if(address){
|
||||
// udata.put("address", address)
|
||||
// }
|
||||
// if(lat2){
|
||||
// udata.put("lng", lng)
|
||||
// udata.put("lat", lat)
|
||||
// udata.put("lng2", lng2)
|
||||
// udata.put("lat2", lat2)
|
||||
// }
|
||||
//
|
||||
// if(alias){
|
||||
// udata.put("alias", alias)
|
||||
// }
|
||||
// if(udata){
|
||||
// index++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// println index+ " "+dist.city +" "+dist.name+" "+udata
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
//def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx"
|
||||
@@ -405,9 +493,51 @@ ExcelUtils.multiThreadingModify("D:\\
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// if(d['K']){
|
||||
// def formatAddress = AddressUtil.formatAddress(d['B'], d['K'])
|
||||
// def address = null
|
||||
// if(formatAddress && formatAddress.endsWith("号")) {
|
||||
// address = formatAddress
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def data = new BasicDBObject();
|
||||
// data.put("m_date", new Date())
|
||||
// if(d['M']){
|
||||
// long glat = d['M'].toString().toDouble() * 10000000000L
|
||||
// long glng = d['L'].toString().toDouble() * 10000000000L
|
||||
// data.put("lat",glat)
|
||||
// data.put("lng",glng)
|
||||
//
|
||||
// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
|
||||
// if(baidu){
|
||||
// long blng = baidu.x * 10000000000L
|
||||
// long blat = baidu.y * 10000000000L
|
||||
// data.put("lat2",blat)
|
||||
// data.put("lng2",blng)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// if(address){
|
||||
// data.put("address", address)
|
||||
// }
|
||||
// index++
|
||||
// println index +" "+d['B'] + " "+data
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
def findDealCount(sqlInstance, def startDate, def endDate){
|
||||
def sql = "select cityname,count(*) as count from bigdata_deal_gj_test where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
|
||||
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
|
||||
def count = 0
|
||||
|
||||
def map = [:]
|
||||
@@ -450,6 +580,18 @@ def findCount(cityid, sqlInstance){
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findDistCount(commid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findFiledCount(cityid, field, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null";
|
||||
def count = 0
|
||||
@@ -567,6 +709,10 @@ def findFiledCount(cityid, field, sqlInstance){
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def isInArea(lng, lat, points){
|
||||
List<Point> pointList = InsidePolygon.createPoint2(points);
|
||||
//Point point = new Point(lng, lat);
|
||||
@@ -636,6 +782,23 @@ def getStreetInfo(def city, def glocation){
|
||||
//}
|
||||
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
|
||||
|
||||
//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
|
||||
//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
|
||||
//
|
||||
//def keys = []
|
||||
//for(def d : datas2){
|
||||
// keys << d['A']+"##"+d['C']+"##"+d['B']
|
||||
//}
|
||||
//
|
||||
//def index = 0
|
||||
//for(def d : datas1){
|
||||
// def k = d['A']+"##"+d['C']+"##"+d['B']
|
||||
// if(!keys.contains(k)){
|
||||
// index ++
|
||||
// println index +" "+k
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+138
-18
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202107
|
||||
def endDate = 202109
|
||||
def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202112
|
||||
def endDate = 202202
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -33,7 +33,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
@@ -48,24 +48,22 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||
for(def city : citys){
|
||||
index++
|
||||
// def statMap = getDistrictStatByCity(city, weekend)
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
//
|
||||
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
// def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
// def rets = createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
// rows.addAll(rets)
|
||||
def rets = createData3(cityCodeDatas.get(city))
|
||||
rows.addAll(rets)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
|
||||
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
|
||||
// def rets = createData3(cityCodeDatas.get(city))//新第三方均价表,要做2步,麻烦
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size() +" "+rows.size()
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_THIRD_PARTY_PRICE_GJ.dat"
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
@@ -183,6 +181,127 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def allRows = []
|
||||
for(def d : dists){
|
||||
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def dataMap = [:]
|
||||
dataMap.COMMID = d._id.toString()
|
||||
dataMap.COMMNAME = d.name
|
||||
dataMap.CITYID = cityCodeDatas.get(d.city)
|
||||
dataMap.CITYNAME = d.city
|
||||
dataMap.DISTID = data.district_code
|
||||
dataMap.DISTNAME = data.district
|
||||
dataMap.YEAR_MONTH = sDate
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
price_map.anjuke_avg_price = third.anjuke_esf_price
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
price_map.fang_avg_price = third.soufang_esf_price
|
||||
}
|
||||
if(third.lianjia_esf_price){
|
||||
price_map.lianjia_avg_price = third.lianjia_esf_price
|
||||
}
|
||||
if(third.cityhouse_esf_price){
|
||||
price_map.cityhouse_avg_price = third.cityhouse_esf_price
|
||||
}
|
||||
if(third.maitian_esf_price){
|
||||
price_map.maitian_avg_price = third.maitian_esf_price
|
||||
}
|
||||
if(third.beike_esf_price){
|
||||
price_map.ke_avg_price = third.beike_esf_price
|
||||
}
|
||||
if(third.get("365house_esf_price")){
|
||||
price_map.taofang_avg_price = third.get("365house_esf_price")
|
||||
}
|
||||
if(third.zhongyuan_esf_price){
|
||||
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
|
||||
}
|
||||
if(third.tuboshi_esf_price){
|
||||
price_map.tbs_avg_price = third.tuboshi_esf_price
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
price_map.qfang_avg_price = third.qfang_esf_price
|
||||
}
|
||||
if(third.fangtan_esf_price){
|
||||
price_map.fangtan_avg_price = third.fangtan_esf_price
|
||||
}
|
||||
if(third.fangduoduo_esf_price){
|
||||
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
|
||||
def rets = []
|
||||
for (def d : dists) {
|
||||
@@ -208,7 +327,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}else if(k=="lianjia_esf_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_esf_price"){
|
||||
web = "城市房产"
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_esf_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="beike_esf_price"){
|
||||
@@ -343,7 +462,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "城市房产"
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
@@ -383,6 +502,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
|
||||
+52
-17
@@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
@@ -28,7 +29,7 @@ class TradeStatLog {
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -39,14 +40,16 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
// task.exportTrade()
|
||||
// task.deleteDistrict()
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
// task.statTableCount()
|
||||
// task.filterTrade()
|
||||
|
||||
|
||||
}
|
||||
@@ -72,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-08-01"
|
||||
def date = "2021-09-01"
|
||||
def endDate = "2021-10-01"
|
||||
def newDate = "2021-11-01"
|
||||
def startDate = "2021-12-01"
|
||||
def date = "2022-01-01"
|
||||
def endDate = "2022-02-01"
|
||||
def newDate = "2022-03-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -128,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\10月成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -478,16 +481,18 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-09-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd"))
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
// def trades = getYouDaData()
|
||||
|
||||
//def statMap = getDistrictStatByCity(city, weekend)
|
||||
def nameMap = getAllDistrict(city)
|
||||
|
||||
|
||||
def rMap = findDealIds(city);
|
||||
def rMap = findDealIds(city, startDate, endDate);
|
||||
def ids = rMap.ids
|
||||
def details = rMap.detail
|
||||
|
||||
@@ -546,8 +551,10 @@ class TradeStatLog {
|
||||
reason = "数据重复"
|
||||
}
|
||||
}
|
||||
if(!reason && t.area<20){
|
||||
reason = "面积过小"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
|
||||
//rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type]
|
||||
@@ -556,8 +563,7 @@ class TradeStatLog {
|
||||
}
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\成交未入库记录(9月).xlsx", rets,"数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
@@ -613,8 +619,8 @@ class TradeStatLog {
|
||||
return trades
|
||||
}
|
||||
|
||||
def findDealIds(city){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-09-01','yyyy-mm-dd') and dealdate<to_date('2021-10-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def findDealIds(city, startDate, endDate){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
@@ -638,7 +644,7 @@ class TradeStatLog {
|
||||
public static def findTradesDetail(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("district_name","百合花园")
|
||||
// q.put("district_name","长隆住宅小区")
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
@@ -693,4 +699,33 @@ class TradeStatLog {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def deleteDealData2(){
|
||||
//sqlInstance.eachRow("select dealid from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-MM-dd')",{r->
|
||||
sqlInstance.executeUpdate("delete from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-MM-dd')")
|
||||
// })
|
||||
|
||||
|
||||
}
|
||||
|
||||
def filterTrade(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d->
|
||||
if(!d['J']){
|
||||
def count = findTradeById(d['A'])
|
||||
if(count){
|
||||
d['J'] = "其它原因过滤"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findTradeById(id){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(id))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class CcbDistrictToDB {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
def run1(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['C']
|
||||
def alias = []
|
||||
if(d['F']){
|
||||
if(name != d['F']){
|
||||
alias << d['F']
|
||||
}
|
||||
}
|
||||
|
||||
def formatAddress = null
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['G'])
|
||||
}
|
||||
def address = null
|
||||
if(d['D']){
|
||||
address = d['D']
|
||||
}
|
||||
if(d['E']){
|
||||
address = address + d['E']
|
||||
}
|
||||
|
||||
if(!formatAddress && address){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], address)
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['H']){
|
||||
glat = d['I'].toString().toDouble() * 10000000000L
|
||||
glng = d['H'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['H'].toString().toDouble(), d['I'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
}
|
||||
}
|
||||
}
|
||||
// }
|
||||
},0);
|
||||
}
|
||||
|
||||
def run(){
|
||||
def files = ["北京融合匹配_1215.xlsx","广州融合匹配_1215.xlsx","杭州融合匹配_1216.xlsx","合肥融合匹配_1216.xlsx","深圳融合匹配_1215.xlsx","石家庄融合匹配_1216.xlsx","苏州融合匹配_1216.xlsx","武汉融合匹配_1216.xlsx","西安融合匹配_1216.xlsx","重庆融合匹配_1215.xlsx"]
|
||||
def index = 0
|
||||
for(def f : files){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\融合\\10个城市新增小区\\" + f, { d ->
|
||||
// if(d['I']=="棠德花苑新社区"){
|
||||
def city = d['A']
|
||||
def region = d['C']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
|
||||
def name = ""
|
||||
def alias = []
|
||||
if(d['I']){
|
||||
name = nameFileter(d['I'])
|
||||
if(name != d['I']){
|
||||
alias << d['I']
|
||||
}
|
||||
}
|
||||
if(name){
|
||||
if(d['D']!=name && !alias.contains(d['D'])){
|
||||
alias << d['D']
|
||||
}
|
||||
}else{
|
||||
name = d['D']
|
||||
}
|
||||
|
||||
def formatAddress = null
|
||||
if(d['J']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['J'])
|
||||
}
|
||||
if(!formatAddress && d['E']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['E'])
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['K'].toString().toDouble() * 10000000000L
|
||||
glng = d['L'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['K'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}else{
|
||||
if(d['F']){
|
||||
def loactionMap = GeoUtils.gps84ToGcj02(d['F'], d['G'])
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(loactionMap){
|
||||
lng = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
lat = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
}
|
||||
def LatLon = GeoHelper.GetLatLon(lat, lng, 0.01, 180);
|
||||
def distance = GeoHelper.GetDistance(lat, lng, LatLon.Lat, LatLon.Lon)
|
||||
|
||||
glat = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lat, 6) * 10000000000L
|
||||
glng = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lon, 6) * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(LatLon.Lon, LatLon.Lat, 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
}
|
||||
}
|
||||
}
|
||||
// }
|
||||
},0);
|
||||
}
|
||||
}
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
}
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
class DistrictMarge {
|
||||
static main(args) {
|
||||
def task = new DistrictMarge()
|
||||
//task.run()
|
||||
task.repeatAddress()
|
||||
}
|
||||
|
||||
def repeatAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
|
||||
findRepead(c, districts)
|
||||
}
|
||||
}
|
||||
def findRepead(city, districts){
|
||||
def map = [:]
|
||||
for(def d : districts){
|
||||
if(d.name_cat == 0 && d.lng){
|
||||
if(d.address && d.address.contains("号")){
|
||||
def list = map.get(d.address)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(d.address, list)
|
||||
}
|
||||
list << d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def newMap = [:]
|
||||
map.each {k,vals->
|
||||
if(vals.size()>=2){
|
||||
vals.sort{o1,o2->
|
||||
return o1.name.size() <=> o2.name.size()
|
||||
}
|
||||
newMap.put(k,vals)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
newMap.each {k,vals->
|
||||
def names = []
|
||||
for(def d : vals){
|
||||
def newKey = d.name
|
||||
if(d.name.contains("小区")){
|
||||
newKey = d.name.toString().replace("小区", "")
|
||||
}
|
||||
if(!names.contains(newKey)){
|
||||
names << newKey
|
||||
}else{
|
||||
d.flag = "删除"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId( d._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def values = newMap.values()
|
||||
def allRets = []
|
||||
for(def list : values){
|
||||
allRets.addAll(list)
|
||||
}
|
||||
def results = []
|
||||
def delcount = 0
|
||||
for(def r : allRets){
|
||||
def rows = []
|
||||
rows << r._id.toString()
|
||||
rows << r.city
|
||||
rows << r.name
|
||||
rows << r.address
|
||||
rows << r.name_cat
|
||||
if(r.flag){
|
||||
rows << r.flag
|
||||
delcount++
|
||||
}
|
||||
results << rows
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
|
||||
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def rets = getDatas()
|
||||
def deleteIndex = 0
|
||||
def updateIndex = 0
|
||||
rets.each{amap_id, valus->
|
||||
def updateObj = null
|
||||
def addAlias = []
|
||||
String formatAddress = null
|
||||
Long lat2 = null
|
||||
Long lng2 = null
|
||||
Long lat = null
|
||||
Long lng = null
|
||||
def amapName = null
|
||||
for(def i=0;i<valus.size();i++){
|
||||
def obj = valus[i]
|
||||
if(obj['G'] && !formatAddress){
|
||||
formatAddress = AddressUtil.formatAddress(obj['B'], obj['G'])
|
||||
}
|
||||
|
||||
if(obj['H'] && !lat){
|
||||
lat = obj['H'].toString().toDouble() * 10000000000L;
|
||||
lng = obj['I'].toString().toDouble() * 10000000000L;
|
||||
def json = BaiDuApiUtils.geoconv(obj['I'].toString().toDouble() , obj['H'].toString().toDouble(), 3)
|
||||
if(json != null && json.containsKey("x")) {
|
||||
lat2 = json.get("y") * 10000000000L
|
||||
lng2 = json.get("x") * 10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
if(obj['F'] && !amapName){
|
||||
amapName = obj['F']
|
||||
}
|
||||
if(i==0){
|
||||
updateObj = obj
|
||||
}else{
|
||||
if(obj['D']){
|
||||
addAlias << obj['D']
|
||||
}
|
||||
//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 8:建行整理删除(单一来源或其它) 9-高德比对王一期不存在删除 10:小区重复合并删除 11:人工整理删除
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 10)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(obj['A']));
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
deleteIndex++
|
||||
println "删除:"+obj['A']
|
||||
}
|
||||
}
|
||||
|
||||
if(updateObj){
|
||||
def dist = CommonDatautils.findDistrictById(updateObj['A'])
|
||||
if(dist){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
for(def n : addAlias){
|
||||
if(!alias.contains(n) && dist.name!=n){
|
||||
alias << n
|
||||
}
|
||||
}
|
||||
if(amapName && !alias.contains(amapName) && amapName!=dist.name){
|
||||
alias << amapName
|
||||
}
|
||||
|
||||
def mudata = new BasicDBObject()
|
||||
if(lat){
|
||||
mudata.put("lat", lat)
|
||||
mudata.put("lng", lng)
|
||||
mudata.put("lat2", lat2)
|
||||
mudata.put("lng2", lng2)
|
||||
}
|
||||
if(formatAddress){
|
||||
mudata.put("address", formatAddress)
|
||||
}
|
||||
if(alias){
|
||||
mudata.put("alias", alias)
|
||||
}
|
||||
|
||||
if(mudata){
|
||||
mudata.put("m_date",new Date())
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(updateObj['A']));
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', mudata))
|
||||
updateIndex++
|
||||
|
||||
println "修改:"+updateObj['A'] + " "+ mudata
|
||||
}else{
|
||||
println amap_id+" 此组无更新"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "组:"+rets.size()
|
||||
println "删:"+deleteIndex
|
||||
println "修:"+updateIndex
|
||||
}
|
||||
|
||||
def getDatas(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕)(1).xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def rets = map.get(d['E'])
|
||||
if(!rets){
|
||||
rets = []
|
||||
map.put(d['E'], rets)
|
||||
}
|
||||
rets << d;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
map.each{k,v->
|
||||
Collections.sort(v,new Comparator<Object>(){
|
||||
public int compare(def o1, def o2) {
|
||||
|
||||
if(o1['D'].toString().length() > o2['D'].toString().length()){
|
||||
return 1;
|
||||
}
|
||||
if(o1['D'].toString().length() == o2['D'].toString().length()){
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+598
@@ -0,0 +1,598 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", ZS_MONGODB_URL)
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
|
||||
if(d['B']){
|
||||
//def avg_price =CommonDatautils.queryCityPrice(d['B'])
|
||||
//d['C'] = avg_price
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("area_name",new BasicDBObject( '$regex' , d['B'].replace("州","")+/.*$/))
|
||||
// def obj = COMMUNITY_CONN.findOne(q)
|
||||
// if(obj){
|
||||
// d['D'] = obj.area_code
|
||||
// println obj.area_name +" "+obj.area_code
|
||||
// }
|
||||
|
||||
//def obj = findDistrictPriceCount(d['B'], DateUtils.parseFromTightByPattern("2022-03-13","yyyy-MM-dd"), CITY_INFO)
|
||||
// d['D'] = CommonDatautils.findAllDistrictCount("district", d['B'])
|
||||
// d['E'] = findHouseCount(d['B'], 20210101, 20211231)
|
||||
//
|
||||
// def startDate = DateUtils.parseFromTightByPattern("2021-01-01", "yyyy-MM-dd")
|
||||
// def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
// def deals = findTradesDetailById(d['B'], startDate,endDate, DEAL_PRICE_COLL)
|
||||
// d['F'] = deals.size()
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
d['G'] = findCcbDistCount(cityCodeDataMap.get(d['B']), sqlInstance)
|
||||
}
|
||||
},0)
|
||||
|
||||
def findCcbDistCount(cityid, sqlInstance){
|
||||
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityId='${cityid}' and status=0 ";
|
||||
def total = null
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
total = r.count
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetailById(city, startDate, endDate, DEAL_PRICE_COLL){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
def findDistrictPriceCount(def city, def weekend, def CITY_INFO){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def obj = CITY_INFO.findOne(city)
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']) {
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']);
|
||||
// if(!rMap && d['G']){
|
||||
// rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['G'], d['H']);
|
||||
// }
|
||||
// if(rMap){
|
||||
// d['O'] = rMap.city
|
||||
// d['P'] = rMap.region
|
||||
// d['Q'] = rMap.name
|
||||
// d['R'] = rMap.address
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']){
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'],"district")
|
||||
// if(dist){
|
||||
// d['K']="完全一致"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'],"district")
|
||||
// if(dist){
|
||||
// d['K']="映射别名"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(rets){
|
||||
// d['K']="别名"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K'] && d['G']){
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['G'],"district")
|
||||
// if(dist){
|
||||
// d['K']="完全一致(高德)"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
|
||||
// if(dist){
|
||||
// d['K']="映射别名(高德)"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(rets){
|
||||
// d['K']="别名(高德)"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤小区融合情况_1209.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['F'], ["address","lat","lng"])
|
||||
// if(dist){
|
||||
// d['Z'] = dist.address
|
||||
// if(dist.lat){
|
||||
// d['AA'] = dist.lat/10000000000.0
|
||||
// d['AB'] = dist.lng/10000000000.0
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
|
||||
//def cityDataMap = [:]
|
||||
//for(def row : cityDatas){
|
||||
// if(row['D']){
|
||||
// cityDataMap.put(row['A']+"##"+row['B']+"##"+row['C'], row)
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据_1529.xlsx",{d->
|
||||
//
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// def key = d['A']+"##"+newRegion+"##"+d['C']
|
||||
// println key
|
||||
// def datas = cityDataMap.get(key)
|
||||
// if(datas){
|
||||
// d['F'] = datas['D']
|
||||
// d['G'] = datas['E']
|
||||
// d['H'] = datas['F']
|
||||
// d['I'] = datas['G']
|
||||
// d['J'] = datas['H']
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def city = "广州"
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(city, d['A'])
|
||||
// if(newRegion){
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
|
||||
// if(rMap){
|
||||
// d['C'] = rMap.region
|
||||
// d['D'] = rMap.name
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(city, d['E'])
|
||||
// if(newRegion){
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['F'], null);
|
||||
// if(rMap){
|
||||
// d['L'] = rMap.region
|
||||
// d['M'] = rMap.name
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",0, 1)
|
||||
//def idMaps = [:]
|
||||
//for(def d : datas){
|
||||
// if(d['L']){
|
||||
// idMaps.put(d['L']+"##"+d['M'], d)
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// if(d['C']){
|
||||
// def key = d['C']+"##"+d['D']
|
||||
// def rows = idMaps.get(key)
|
||||
// if(rows){
|
||||
// d['E'] = rows['D']
|
||||
// d['F'] = rows['E']
|
||||
// d['G'] = rows['F']
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例(合并).xlsx",{d->
|
||||
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
|
||||
// if(obj){
|
||||
//// def year = CommonUtils.getDistrictAge(obj)
|
||||
//// if(year){
|
||||
//// d['H'] = year
|
||||
//// }
|
||||
//// if(obj.alias){
|
||||
//// d['K'] = obj.alias.join(" ")
|
||||
//// }
|
||||
// if(obj.mapping_alias){
|
||||
// d['X'] = obj.mapping_alias.join(" ")
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
|
||||
//def scoreMap = [:]
|
||||
//for(def d : data1s){
|
||||
// scoreMap.put(d['B'], d)
|
||||
//}
|
||||
//
|
||||
//def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
|
||||
//def levelMap = [:]
|
||||
//for(def d : data2s){
|
||||
// levelMap.put(d['E'], d)
|
||||
//}
|
||||
//
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// if(d['E']){
|
||||
// def row = scoreMap.get(d['E'])
|
||||
// if(row){
|
||||
// d['I'] = row['G']
|
||||
// d['J'] = row['H']
|
||||
// d['K'] = row['I']
|
||||
// d['L'] = row['J']
|
||||
// d['M'] = row['K']
|
||||
// d['N'] = row['L']
|
||||
// d['O'] = row['M']
|
||||
// }
|
||||
//
|
||||
// def data = levelMap.get(d['E'])
|
||||
// if(data){
|
||||
// d['P'] = data['G']
|
||||
// d['Q'] = data['H']
|
||||
// d['R'] = data['I']
|
||||
// d['S'] = data['J']
|
||||
// d['T'] = data['K']
|
||||
// d['U'] = data['L']
|
||||
// d['V'] = data['M']
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\成交\\福州成交.xlsx",{d->
|
||||
// def o = new BasicDBObject()
|
||||
// o.area = d['D'].toDouble()
|
||||
// o.avg_price = d['F'].toInteger()
|
||||
// o.city = "福州"
|
||||
// o.district_name = d['C']
|
||||
// o.region = d['B']
|
||||
// o.source = "贝壳找房"
|
||||
// o.trade_date = DateUtils.parseFromTightYYYYMMDD(d['M'])
|
||||
// o.total_price = d['E'].toInteger()*10000
|
||||
// if(d['I']){
|
||||
// o.hall = d['I'].toInteger()
|
||||
// }
|
||||
//
|
||||
// o.m_date = new Date()
|
||||
// if(d['H']){
|
||||
// o.room = d['H'].toInteger()
|
||||
// }
|
||||
//
|
||||
// o.create_date = new Date()
|
||||
// o.status = 2
|
||||
// DEAL_PRICE.save(o)
|
||||
//
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// if(d['K'] && d['K']=="别名"){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
|
||||
// def dist = null
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// }
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// if(d['K'] && d['K']=="映射别名(高德)" && d['G']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// }
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// if(d['K'] && d['K']=="别名(高德)" && d['G']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||
// def dist = null
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// }
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
|
||||
//def rets = []
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d ->
|
||||
// if(d['A'] && d['E'].toString().contains("鑫康花园北区")){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['E'])){
|
||||
// alias << d['E']
|
||||
// }
|
||||
// if(alias && dist.mapping_alias!=alias){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// rets << [dist._id.toString(), d['E']]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据")
|
||||
|
||||
|
||||
//def rets = []
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def dists = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = null
|
||||
// if(dists && dists.size()==1){
|
||||
// dist = dists[0]
|
||||
// }
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['D'])){
|
||||
// alias << d['D']
|
||||
// }
|
||||
// if(alias && dist.mapping_alias!=alias){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// rets << [dist._id.toString(), d['D']]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据")
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d ->
|
||||
// if(d['A']) {
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if (newRegion) {
|
||||
// def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
|
||||
// if (dist) {
|
||||
// rets << [dist._id.toString(), d['C']]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(dist){
|
||||
// d['E'] = dist.city
|
||||
// d['F'] = dist.region
|
||||
// d['G'] = dist.name
|
||||
// d['H'] = dist.estate_type2
|
||||
// d['I'] = dist.name_cat
|
||||
// d['J'] = dist.status
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d->
|
||||
// if(d['B']) {
|
||||
// def sql = "select count(*) as count from bigdata_community_gj where commid='${d['B']}'";
|
||||
// def count = 0;
|
||||
// sqlInstance.eachRow(sql, { r ->
|
||||
// count = r.count
|
||||
// })
|
||||
// if(count>=1){
|
||||
// d['J'] = "oracle存在"
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
// if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(alias && !alias.contains(d['C'])){
|
||||
// index++
|
||||
// println index
|
||||
// alias << d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
|
||||
// if(d['A']){
|
||||
// try{
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
|
||||
// if(dist){
|
||||
// d['B'] = dist.city
|
||||
// d['C'] = dist.region
|
||||
// d['D'] = dist.name
|
||||
//
|
||||
// def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
|
||||
// d['E'] = price
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
// println d['A']
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def mappingAlias = dist.mapping_alias
|
||||
// if(mappingAlias){
|
||||
// if(mappingAlias.contains(d['B'])){
|
||||
// mappingAlias.remove(d['B'])
|
||||
// DBObject udata = new BasicDBObject();
|
||||
// udata.put("mapping_alias", mappingAlias);
|
||||
// udata.put("m_date", new Date());
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d['A']+"无映射别名"
|
||||
// }
|
||||
// }else{
|
||||
// println d['A']+"映射别名为空"
|
||||
// }
|
||||
// }else{
|
||||
// println d['A']+"没找到"
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class MapAddressMatch {
|
||||
static main(args) {
|
||||
def task = new MapAddressMatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "北京"
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d ->
|
||||
index++
|
||||
println index +" "+ d['D']+" "+d['E']
|
||||
if(!d['G']){
|
||||
// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德")
|
||||
// if(gdData){
|
||||
// d['F'] = gdData.district
|
||||
// d['G'] = gdData.name
|
||||
// d['H'] = gdData.address
|
||||
// d['I'] = gdData.type
|
||||
// d['J'] = "高德"
|
||||
// }else{
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E'])
|
||||
def map = findStandardObject(city,d['D'],d['E'], rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']);
|
||||
map = findStandardObject(city,d['D'],d['E'], rets);
|
||||
}
|
||||
if(map){
|
||||
if(map.adname){
|
||||
d['F'] = map.adname
|
||||
}else if(map.region){
|
||||
d['F'] = map.region
|
||||
}
|
||||
d['G'] = map.name
|
||||
d['H'] = map.address
|
||||
if(map.type){
|
||||
d['I'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['I'] = map.estateType
|
||||
}
|
||||
|
||||
d['J'] = map.source
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
|
||||
continue;
|
||||
}
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class MatchDealPriceRegion {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchDealPriceRegion();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\成交\\部分未匹配成交.xlsx",{d->
|
||||
|
||||
def newRegion = null
|
||||
if(d['C']){
|
||||
newRegion = RegionData.formatRegion(d['B'], d['C'])
|
||||
}
|
||||
if(!newRegion){
|
||||
index++
|
||||
newRegion = findRegionByThirdPrice(d['B'], d['D']);
|
||||
if(!newRegion){
|
||||
def rets = CommonDatautils.findDistrictsByName(d['B'], d['D'],"district")
|
||||
if(!rets){
|
||||
rets = CommonDatautils.findDistrictByAlias(d['B'], null,d['D'],"district")
|
||||
}
|
||||
if(rets && rets.size()==1){
|
||||
newRegion = rets[0].region
|
||||
}
|
||||
}
|
||||
if(newRegion){
|
||||
def query = new BasicDBObject()
|
||||
query.put("_id", new ObjectId(d['A']))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", newRegion)
|
||||
data.put("m_date", new Date())
|
||||
println index+" "+d['A']+" "+d['C']+" "+newRegion
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findRegionByThirdPrice(def city, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
def dbcursor = collection.find(q).sort(new BasicDBObject("s_date",-1)).limit(10)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def count = map.get(d.region)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(d.region, count+1)
|
||||
}
|
||||
if(map){
|
||||
if(map.size()>1){
|
||||
map = map.sort(){ a, b ->
|
||||
b.value.compareTo(a.value)
|
||||
}
|
||||
}
|
||||
def key = map.keySet().getAt(0)
|
||||
return key
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
+581
@@ -0,0 +1,581 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class MatchMapData {
|
||||
|
||||
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchMapData()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.exportHouse()
|
||||
// task.addDistrictLevel()
|
||||
task.run4()
|
||||
|
||||
}
|
||||
|
||||
def run4() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
|
||||
def regionCode = null
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
regionCode = getRegionCode(newRegion)
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
d['F'] = "完全一致"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
} else {
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
d['F'] = "映射别名"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
} else {
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (rets) {
|
||||
if(rets.size()>1){
|
||||
d['F'] = "别名-多个"
|
||||
}else{
|
||||
d['F'] = "别名"
|
||||
}
|
||||
}
|
||||
if (rets) {
|
||||
dist = rets[0]
|
||||
}
|
||||
if (dist) {
|
||||
d['F'] = "完全一致"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def run3(){
|
||||
|
||||
def files = ["合肥融合匹配_1215.xlsx"]
|
||||
|
||||
for(def fn : files){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合匹配房源\\"+fn,{d->
|
||||
def city = d['A']
|
||||
def region = d['D']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def count = findHouses2(city, newRegion, "sell", 20160101, d['E']);
|
||||
if(count){
|
||||
d['O'] = "有"
|
||||
}else{
|
||||
count = findThirdParty(city, newRegion, d['E']);
|
||||
if(count){
|
||||
d['O'] = "有"
|
||||
}else{
|
||||
count = findThirdParty(city, region, d['E']);
|
||||
if(count){
|
||||
d['O'] = "有"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
println dist.name
|
||||
},0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
//"合肥","北京","广州","杭州","上海","深圳","石家庄","天津","武汉",
|
||||
def files = ["苏州"]
|
||||
for(def f : files){
|
||||
def city = f
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d->
|
||||
if(d['E']){
|
||||
def region = d['E']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def rMap = match(city, newRegion, d)
|
||||
if(rMap){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(rMap.city, rMap.region, rMap.name)
|
||||
d['M'] = rMap.region
|
||||
d['N'] = rMap.name
|
||||
d['O'] = rMap.address
|
||||
if(priceObj){
|
||||
d['P'] = priceObj.avg_price
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(rMap.id)
|
||||
if(dist){
|
||||
d['Q'] = dist.lng/10000000000.0
|
||||
d['R'] = dist.lat/10000000000.0
|
||||
def count = findHouses(dist.city, dist.region, "rent", 20210801, 20210901, dist.name, dist.alias);
|
||||
if(count){
|
||||
d['V'] = "有"
|
||||
}
|
||||
}
|
||||
|
||||
if(d['I']){
|
||||
def loactionMap = GeoUtils.gps84ToGcj02(d['H'], d['I'])
|
||||
if(loactionMap){
|
||||
d['S'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
d['T'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
}
|
||||
|
||||
def distance = GeoHelper.GetDistance(d['R'], d['Q'], d['T'], d['S'])
|
||||
d['U'] = distance
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
def match(city, region, d){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, region, d['F'], null);
|
||||
if(!rMap && d['G']){
|
||||
def addrFlag = d['G'].toString().matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"广州",
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def files = ["合肥","广州","上海","北京","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"]
|
||||
for(def f : files){
|
||||
def city = f.replace("-2","")
|
||||
def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365)
|
||||
def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear))
|
||||
def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def priceDatas = addPriceData(dists, priceMap);
|
||||
def levelMap = calcuteLevel(priceDatas);
|
||||
|
||||
def datas = new CopyOnWriteArrayList()
|
||||
datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型","评级","均价"]
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d->
|
||||
def regionCode = null
|
||||
if(d['E']){
|
||||
def region = d['E']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
regionCode = getRegionCode(newRegion)
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, d['F'],"district")
|
||||
if(dist){
|
||||
d['K']="完全一致"
|
||||
d['L'] = dist.name_cat
|
||||
}else{
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(city, newRegion, d['F'],"district")
|
||||
if(dist){
|
||||
d['K']="映射别名"
|
||||
d['L'] = dist.name_cat
|
||||
}else{
|
||||
def rets = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district")
|
||||
if(rets){
|
||||
d['K']="别名"
|
||||
}
|
||||
if(rets){
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(dist){
|
||||
def nearDbo = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
|
||||
def lastMonth = DateUtils.parseFromTightYYYYMMDD("2021-11-30")
|
||||
def sqmetre = CommonDatautils.findSqmetreRent(dist.city, dist.region, dist.name, lastMonth)
|
||||
if(nearDbo && districtPriceMap){
|
||||
def priceObj = districtPriceMap.get(dist.region+"#"+dist.name)
|
||||
if(priceObj){
|
||||
def markup = (nearDbo.avg_price - priceObj.avg_price)*100/priceObj.avg_price
|
||||
d['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, markup)
|
||||
}else{
|
||||
// FangjiaClientUtils.sendDistrictPriceCalculate(dist.city, dist.region, dist.name, 60);
|
||||
// println dist.city+" "+ dist.region+ " "+ dist.name
|
||||
}
|
||||
}
|
||||
if(nearDbo && sqmetre){
|
||||
def prc = nearDbo.avg_price/sqmetre.avg_price
|
||||
d['N'] = prc.toInteger()
|
||||
}
|
||||
d['O'] = dist.estate_type2
|
||||
|
||||
|
||||
def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
if(dist.name == "绿苑公寓"){
|
||||
println dbo
|
||||
}
|
||||
if(dbo){
|
||||
d['P'] = dbo.level
|
||||
d['Q'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['M'] = "区域无法标准化"
|
||||
}
|
||||
}
|
||||
|
||||
datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O'], d['P'], d['Q']]
|
||||
|
||||
},0)
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+city+".xlsx", datas, "数据")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def getRegionCode(def region){
|
||||
def spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(region, " ")
|
||||
def arrs = spell.split(" ")
|
||||
def s = new StringBuffer()
|
||||
for(def ar : arrs){
|
||||
s.append(ar.substring(0,1).toUpperCase())
|
||||
}
|
||||
return s.toString();
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses2(def city, def region, def cat, def startDate, def name){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
def alias = []
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
if(name.contains("号小区")){
|
||||
def newName = name.replace("号小区","号")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
println "[房源] "+city +" "+region+" "+ name +" "+houses.size()
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
def findThirdParty(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("region",region)
|
||||
q.put("name",name)
|
||||
def count = thirdPartyColl.count(q)
|
||||
println "[第三方] "+city +" "+region+" "+ name +" "+count
|
||||
return count
|
||||
}
|
||||
|
||||
def exportHouse(){
|
||||
//"合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆",
|
||||
def files = ["苏州","合肥"]
|
||||
for(def f : files) {
|
||||
def city = f
|
||||
|
||||
def rets = []
|
||||
rets << ["建行小区id", "房价网小区id","城市","区域","楼盘名称","单价","月租金","面积","室","厅","卫","出租方式","朝向","楼屋","装修","发布时间"]
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\" + f + ".xlsx", { d ->
|
||||
def flag = d['K']
|
||||
if(flag && flag =="完全一致" && !d['J']){
|
||||
def obj = CommonDatautils.findDistrict(city, d['M'], d['N'], "district")
|
||||
if(obj){
|
||||
def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20211101, d['N'], null);
|
||||
if(houses){
|
||||
rets.addAll(houses)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房3\\"+city+"_rent.xlsx", rets, "数据")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses3(def commid, def fjwid, def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def rets = []
|
||||
for(def r : houses){
|
||||
rets << [commid, fjwid, r.city,r.region,r.district_name, r.avg_price, r.total_price, r.area,r.room, r.hall,r.toilet,r.rent_type, r.direction,r.floor_info,r.fitment, r.p_date ]
|
||||
}
|
||||
|
||||
if(rets){
|
||||
println city+" "+ name +" "+rets.size()
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addDistrictLevel(){
|
||||
//
|
||||
def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
|
||||
|
||||
def dataMap = null
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
for(def fn : files){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
def levelMap = calcuteLevel(datas);
|
||||
|
||||
dataMap = getDataMap(fn);
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\评级10级\\"+fn+".xlsx",{d->
|
||||
|
||||
def region = dataMap.get(d['C'])
|
||||
if(region){
|
||||
def newRegion = RegionData.formatRegion(fn, region)
|
||||
if(newRegion){
|
||||
def dist = CommonDatautils.findDistrict(fn, newRegion, d['D'],"district")
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(fn, newRegion, d['D'],"district")
|
||||
if(!dist){
|
||||
def rets = CommonDatautils.findDistrictByAlias(fn, newRegion, d['D'],"district")
|
||||
if(rets){
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if(dist){
|
||||
def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
if(dbo){
|
||||
d['J'] = dbo.level
|
||||
d['K'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
newLists << d;
|
||||
}
|
||||
}
|
||||
if(newLists){
|
||||
newLists.sort {a,b->
|
||||
def v1 = a.avg_price
|
||||
def v2 = b.avg_price
|
||||
return v1.compareTo(v2)
|
||||
}
|
||||
}
|
||||
return newLists;
|
||||
}
|
||||
|
||||
def calcuteLevel(def datas){
|
||||
def total = datas.size()
|
||||
def v = (int)(total/10)
|
||||
def start = 0;
|
||||
def end = 0;
|
||||
def map = [:]
|
||||
for(int i=1;i<=10;i++){
|
||||
end = end + v;
|
||||
if(i == 10){
|
||||
map.put(i, [start, total])
|
||||
}else{
|
||||
map.put(i, [start, end])
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for(def d=0;d<datas.size();d++){
|
||||
def dist = datas[d]
|
||||
|
||||
for (def o in map) {
|
||||
def ranges = o.value
|
||||
def m = ranges[0]
|
||||
def n = ranges[1]
|
||||
if(d==0){
|
||||
if(d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}else{
|
||||
if(d>m && d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def dataMap = [:]
|
||||
for(def d : datas){
|
||||
if(d.name=="绿苑公寓"){
|
||||
println d
|
||||
}
|
||||
dataMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def getDataMap(city){
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+city+".xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['D'], d['E'])
|
||||
}
|
||||
return map
|
||||
|
||||
}
|
||||
def getDistrictLevel(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
return ZS_LEVEL_CONN.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class TradeDistrictToDB {
|
||||
static main(args) {
|
||||
def task = new TradeDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def index = 0
|
||||
def keys = []
|
||||
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
|
||||
if(d['D']){
|
||||
def key = d['A']+"##"+d['D']+"##"+d['C']
|
||||
if(!keys.contains(key)){
|
||||
def formatAddress = null
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['F'])
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['G']){
|
||||
glat = d['H'].toString().toDouble() * 10000000000L
|
||||
glng = d['G'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def name = d['C'].replace("·","").replace("II","二")
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
keys << key
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], d['D'], name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",d['D'])
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['D'], d['C'], 4);
|
||||
println index+" "+data
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run() {
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if (newRegion) {
|
||||
def map = findMapData(city, newRegion, d['C'])
|
||||
if(map){
|
||||
d['D'] = newRegion
|
||||
d['E'] = map.name
|
||||
d['F'] = map.address
|
||||
d['G'] = map.lng
|
||||
d['H'] = map.lat
|
||||
d['I'] = map.lng2
|
||||
d['J'] = map.lag2
|
||||
}
|
||||
}
|
||||
}, 0);
|
||||
}
|
||||
|
||||
def findMapData(def city, def region, name){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
|
||||
def map = findStandardObject2(city, region, name, null, rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(city, region, name);
|
||||
map = findStandardObject2(city, region, name, null, rets);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+52
-70
@@ -32,7 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
public static def batchsize = 1000
|
||||
@@ -42,22 +42,24 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = ["大连","哈尔滨","呼和浩特","厦门"]//CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// for (def city : citys) {
|
||||
// def cityid = cityCodeDataMap.get(city)
|
||||
// def sql ="select count(*) as count from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'"
|
||||
// def sql ="select count(*) as count from BIGDATA_COMMUNITY_GJ where CITYID='${cityid}'"
|
||||
// sqlInstance.eachRow(sql, { r ->
|
||||
// println city +" "+r.count
|
||||
// })
|
||||
// }
|
||||
|
||||
//// addBigdataDistrictFJW()
|
||||
// addBigdataDistrictFJW()
|
||||
|
||||
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
||||
|
||||
def index = 0
|
||||
for (def city : citys) {
|
||||
index++
|
||||
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
|
||||
//
|
||||
// for(int i=0;i<10;i++){
|
||||
@@ -88,23 +90,24 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
|
||||
}
|
||||
}
|
||||
println city +" 小区加载完成 "+districts.size()
|
||||
println index +" "+city +" 小区加载完成 "+districts.size()
|
||||
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
||||
println city +" 1.小区表完成"
|
||||
println index +" "+city +" 1.小区表完成"
|
||||
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
||||
println city +" 2.小区建筑结构表"
|
||||
println index +" "+city +" 2.小区建筑结构表"
|
||||
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
||||
println city +" 3.小区房屋类型表"
|
||||
println index +" "+city +" 3.小区房屋类型表"
|
||||
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
||||
println city +" 4.小区建筑年代表"
|
||||
println index +" "+city +" 4.小区建筑年代表"
|
||||
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
||||
println city +" 5.小区费用表"
|
||||
println index +" "+city +" 5.小区费用表"
|
||||
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||
println city +" 6.小区房屋用途表"
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
|
||||
println city + " 7.地理位置表"
|
||||
println index +" "+city +" 6.小区房屋用途表"
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
||||
println index +" "+ city + " 7.地理位置表"
|
||||
|
||||
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
||||
|
||||
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -551,18 +554,17 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
// }
|
||||
}
|
||||
|
||||
def addBigdataPositionGJ(districts, def cityid){
|
||||
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
|
||||
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : districts) {
|
||||
index++
|
||||
def ret = []
|
||||
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
|
||||
ret << d.cityid
|
||||
ret << "5"
|
||||
ret << d.commid
|
||||
ret << "1"
|
||||
ret << index
|
||||
ret << seqno
|
||||
ret << d.lng
|
||||
ret << d.lat
|
||||
ret << ""
|
||||
@@ -588,49 +590,29 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
|
||||
println "BIGDATA_POSITION_FJW Êý¾ÝÎļþÊä³öÍê³É......"
|
||||
}
|
||||
|
||||
// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
|
||||
//
|
||||
// def sqls = []
|
||||
// def index = 1
|
||||
// for(def d : districts){
|
||||
// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
|
||||
// index++
|
||||
// if(d.createtime){
|
||||
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
|
||||
// }else{
|
||||
// sql = sql + "null,"
|
||||
// }
|
||||
// if(d.updatetime){
|
||||
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
|
||||
// }else{
|
||||
// sql = sql + "null,'${d.sourcecode}')"
|
||||
// }
|
||||
// sqls << sql
|
||||
//
|
||||
// if (sqls.size() >= batchsize) {
|
||||
// try {
|
||||
// sqlInstance.withBatch(batchsize, { stmt ->
|
||||
// sqls.each { def s ->
|
||||
// stmt.addBatch s
|
||||
// }
|
||||
// })
|
||||
//
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// }
|
||||
// sqls.clear()
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// if (sqls.size() > 0) {
|
||||
// sqlInstance.withBatch(100, { stmt ->
|
||||
// sqls.each { def s ->
|
||||
// stmt.addBatch s
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
def findSeqno(cityid, commid, citySeqnoMap){
|
||||
def detailMap = citySeqnoMap.get(cityid)
|
||||
if(detailMap){
|
||||
def seqno = detailMap.get(commid)
|
||||
if(seqno){
|
||||
return seqno
|
||||
}else{
|
||||
seqno = CcbCommonUtils.findMaxSeqno(cityid)
|
||||
if(seqno){
|
||||
seqno = seqno + 1
|
||||
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
def insertSql = "INSERT INTO bigdata_position_seqno(CITYID,COMMID,SEQNO,CREATETIME) VALUES ('${cityid}','${commid}',${seqno},to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'))"
|
||||
sqlInstance.withBatch(batchsize, { stmt ->
|
||||
stmt.addBatch insertSql
|
||||
})
|
||||
return seqno
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println cityid+" 没有该城市的小区序号"
|
||||
}
|
||||
}
|
||||
|
||||
def addBigdataDistrictFJW(){
|
||||
@@ -653,7 +635,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def updateDealHouseStyle(distMap, cityid){
|
||||
def sql = "select dealid,housestyle,buildtype,roomcnt,livingroomcnt,buildyear,address,street,streetno,commid from bigdata_deal_gj_fjw where cityid='${cityid}'";
|
||||
def sql = "select dealid,housestyle,buildtype,roomcnt,livingroomcnt,buildyear,address,street,streetno,commid from bigdata_deal_gj where cityid='${cityid}'";
|
||||
try{
|
||||
def index = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
@@ -695,21 +677,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
def buildyear = null
|
||||
if(dist){
|
||||
// if(!r.address){
|
||||
if(r.street || r.streetno){
|
||||
address = r.street
|
||||
if(r.streetno && r.streetno!="null"){
|
||||
address = address + r.streetno
|
||||
}
|
||||
}else if(r.street){
|
||||
address = dist.address
|
||||
if(r.street || r.streetno){
|
||||
address = r.street
|
||||
if(r.streetno && r.streetno!="null"){
|
||||
address = address + r.streetno
|
||||
}
|
||||
}else if(r.street){
|
||||
address = dist.address
|
||||
}
|
||||
// }
|
||||
if(!r.buildyear && dist.finish_building_date){
|
||||
buildyear = CcbCodeUtils.getBuildYear(dist.finish_building_date);
|
||||
}
|
||||
}
|
||||
if(housetype || buildyear || address){
|
||||
def updateSql = "update bigdata_deal_gj_fjw set housestyle='${housetype}'"
|
||||
def updateSql = "update bigdata_deal_gj set housestyle='${housetype}'"
|
||||
if(buildyear){
|
||||
updateSql = updateSql + ",buildyear=${buildyear}"
|
||||
}
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class CcbRHDealFileCreateBatch extends BaseBatch{
|
||||
static main(args) {
|
||||
runBatch(new CcbRHDealFileCreateBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
|
||||
}
|
||||
}
|
||||
+399
-75
@@ -16,7 +16,7 @@ import java.sql.ResultSetMetaData
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
runBatch(new ExportDistrictRongHe(), args);
|
||||
@@ -24,29 +24,77 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["大连","哈尔滨","呼和浩特","厦门"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
exportBigdataCommbuildstruGJ(cityCode)
|
||||
exportBigdataCommunityGJRH(cityCode)
|
||||
exportBigdataCommhousetypeGJ(cityCode)
|
||||
exportBigdataCommbuildyearGj(cityCode)
|
||||
exportBigdataCommfeeGj(cityCode)
|
||||
exportBigdataCommhouseuseGJ(cityCode)
|
||||
exportBigdataPositionGJ(cityCode)
|
||||
exportBigdataDistrictFJW(cityCode)
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
//按天导(每天一个文件夹)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"))
|
||||
|
||||
exportBigdataDealFJW(cityCode)
|
||||
exportBigdataBuildingFJW(cityCode)
|
||||
if(curDate.toString().endsWith("10")){
|
||||
exportBigdataCommunityGJRH()
|
||||
exportBigdataBuildingFJW()
|
||||
exportBigdataCommfeeGj()
|
||||
|
||||
exportBigdataCommbuildstruGJ()
|
||||
exportBigdataCommhousetypeGJ()
|
||||
exportBigdataCommbuildyearGj()
|
||||
exportBigdataCommhouseuseGJ()
|
||||
exportBigdataPositionGJ()
|
||||
exportBigdataDistrictFJW()
|
||||
exportBigdataAliasFJW()
|
||||
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
exportEvaluateRate()
|
||||
}
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportEvaluateRate()
|
||||
exportOk(curDate)
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
// def runByCity(){
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//
|
||||
//
|
||||
//// //检查ok文件是否上传
|
||||
//// exportAllBigdataDealFJW()
|
||||
//// for(def city : citys) {
|
||||
//// def cityCode = cityCodeDataMap.get(city)
|
||||
////
|
||||
// //按天导(一个文件夹下)
|
||||
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
// //按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
//
|
||||
//// exportBigdataCommunityGJRH(cityCode)
|
||||
//// exportBigdataBuildingFJW(cityCode)
|
||||
//// exportBigdataCommfeeGj(cityCode)
|
||||
////
|
||||
//// exportBigdataCommbuildstruGJ(cityCode)
|
||||
//// exportBigdataCommhousetypeGJ(cityCode)
|
||||
//// exportBigdataCommbuildyearGj(cityCode)
|
||||
//// exportBigdataCommhouseuseGJ(cityCode)
|
||||
//// exportBigdataPositionGJ(cityCode)
|
||||
//// exportBigdataDistrictFJW(cityCode)
|
||||
//// exportBigdataAliasFJW(cityCode)
|
||||
////
|
||||
//// exportBigdataBuildprojFjw(cityCode)
|
||||
//// exportBigdataCommlayoutpicFjw(cityCode)
|
||||
//// }
|
||||
//// exportEvaluateRate()
|
||||
//
|
||||
////
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
//
|
||||
// exportOk()
|
||||
// }
|
||||
|
||||
def exportBigdataCommunityGJRH(cityCode){
|
||||
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'";
|
||||
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -65,28 +113,25 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << r.INTRODUCTION
|
||||
data << ''
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
|
||||
}
|
||||
def exportBigdataCommbuildstruGJ(cityCode){
|
||||
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -100,17 +145,17 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommhousetypeGJ(cityCode){
|
||||
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'";
|
||||
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -124,17 +169,17 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommbuildyearGj(cityCode){
|
||||
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataCommbuildyearGj(){
|
||||
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -148,17 +193,17 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommfeeGj(cityCode){
|
||||
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataCommfeeGj(){
|
||||
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -167,7 +212,12 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.COMPID
|
||||
data << r.FEE
|
||||
data << r.HOUSEUSE
|
||||
data << r.BUILDTYPE
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.FEETYPE
|
||||
data << r.NOTE
|
||||
data << r.REMARK
|
||||
@@ -177,18 +227,18 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommhouseuseGJ(cityCode){
|
||||
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataCommhouseuseGJ(){
|
||||
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -203,17 +253,17 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataPositionGJ(cityCode){
|
||||
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataPositionGJ(){
|
||||
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -232,17 +282,17 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDistrictFJW(cityCode){
|
||||
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataDistrictFJW(){
|
||||
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -260,17 +310,250 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDealFJW(cityCode){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE from BIGDATA_DEAL_FJW where CITYID='${cityCode}'";
|
||||
def exportBigdataAliasFJW(){
|
||||
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.city_id
|
||||
data << r.ca_id
|
||||
data << r.address_id
|
||||
data << r.address_name
|
||||
data << r.status
|
||||
data << r.exp_dt
|
||||
data << r.version
|
||||
data << r.is_key_name
|
||||
data << r.data_source
|
||||
data << r.description
|
||||
data << r.alias_grade
|
||||
data << r.addr_type_cd
|
||||
data << r.ifexamine
|
||||
data << r.ifdisplay
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDealFJW(dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def stop = false
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def index = 0
|
||||
|
||||
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
|
||||
while(!stop){
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(index==1){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
if(curStartDateStr == curSndDateStr){
|
||||
stop = true
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
|
||||
def curEndStr = DateUtils.add(curStartStr, 1)
|
||||
curStartDateStr = curEndStr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataDealFJW2(curDate, dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(curDate.toString().endsWith("28")){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
def exportAllBigdataDealFJW(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -298,7 +581,12 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
data << r.BUILDTYPE
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
@@ -334,25 +622,24 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......"
|
||||
println "BIGDATA_DEAL_FJW 导出完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataBuildingFJW(cityCode){
|
||||
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";
|
||||
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -364,7 +651,12 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
data << r.BUILDTYPE
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.FLOORCNT
|
||||
data << r.UNITCNT
|
||||
@@ -380,12 +672,12 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -413,8 +705,40 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(fileName, curDate)
|
||||
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "EVALUATE_RATE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+747
@@ -0,0 +1,747 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
runBatch(new ExportDistrictRongHeTest(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def fileName = "/ccb_rh/${curDate}"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}"
|
||||
// }
|
||||
CcbFtpTestUtils.delete(fileName)
|
||||
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
|
||||
|
||||
exportBigdataCommunityGJRH()
|
||||
exportBigdataBuildingFJW()
|
||||
exportBigdataCommfeeGj()
|
||||
|
||||
exportBigdataCommbuildstruGJ()
|
||||
exportBigdataCommhousetypeGJ()
|
||||
exportBigdataCommbuildyearGj()
|
||||
exportBigdataCommhouseuseGJ()
|
||||
exportBigdataPositionGJ()
|
||||
exportBigdataDistrictFJW()
|
||||
exportBigdataAliasFJW()
|
||||
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
exportEvaluateRate()
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportOk(curDate)
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
// def runByCity(){
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//
|
||||
//
|
||||
//// //检查ok文件是否上传
|
||||
//// exportAllBigdataDealFJW()
|
||||
//// for(def city : citys) {
|
||||
//// def cityCode = cityCodeDataMap.get(city)
|
||||
////
|
||||
// //按天导(一个文件夹下)
|
||||
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
// //按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
//
|
||||
//// exportBigdataCommunityGJRH(cityCode)
|
||||
//// exportBigdataBuildingFJW(cityCode)
|
||||
//// exportBigdataCommfeeGj(cityCode)
|
||||
////
|
||||
//// exportBigdataCommbuildstruGJ(cityCode)
|
||||
//// exportBigdataCommhousetypeGJ(cityCode)
|
||||
//// exportBigdataCommbuildyearGj(cityCode)
|
||||
//// exportBigdataCommhouseuseGJ(cityCode)
|
||||
//// exportBigdataPositionGJ(cityCode)
|
||||
//// exportBigdataDistrictFJW(cityCode)
|
||||
//// exportBigdataAliasFJW(cityCode)
|
||||
////
|
||||
//// exportBigdataBuildprojFjw(cityCode)
|
||||
//// exportBigdataCommlayoutpicFjw(cityCode)
|
||||
//// }
|
||||
//// exportEvaluateRate()
|
||||
//
|
||||
////
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
//
|
||||
// exportOk()
|
||||
// }
|
||||
|
||||
def exportBigdataCommunityGJRH(cityCode){
|
||||
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.COMMALIAS
|
||||
data << r.DISTID
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.ADDRESS
|
||||
data << r.LANDAREA
|
||||
data << r.BUILDAREA
|
||||
data << r.COMMTYPE
|
||||
data << r.IFNEW
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << ''
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
|
||||
}
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDSTRU
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommhousetypeGJ(cityCode){
|
||||
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.HOUSETYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommbuildyearGj(){
|
||||
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommfeeGj(){
|
||||
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.COMPID
|
||||
data << r.FEE
|
||||
data << r.HOUSEUSE
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.FEETYPE
|
||||
data << r.NOTE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommhouseuseGJ(){
|
||||
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSECNT
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataPositionGJ(){
|
||||
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.OBJECTTYPE
|
||||
data << r.OBJECTID
|
||||
data << r.POSITIONTYPE
|
||||
data << r.SEQNO
|
||||
data << r.LONGITUDE
|
||||
data << r.LATITUDE
|
||||
data << r.ALTITUDE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDistrictFJW(){
|
||||
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.DISTSPELL
|
||||
data << r.CITYID
|
||||
data << r.DISTCODE
|
||||
data << r.SEQ
|
||||
data << r.DISTTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataAliasFJW(){
|
||||
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.city_id
|
||||
data << r.ca_id
|
||||
data << r.address_id
|
||||
data << r.address_name
|
||||
data << r.status
|
||||
data << r.exp_dt
|
||||
data << r.version
|
||||
data << r.is_key_name
|
||||
data << r.data_source
|
||||
data << r.description
|
||||
data << r.alias_grade
|
||||
data << r.addr_type_cd
|
||||
data << r.ifexamine
|
||||
data << r.ifdisplay
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDealFJW(dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def stop = false
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def index = 0
|
||||
|
||||
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
|
||||
while(!stop){
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(index==1){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
if(curStartDateStr == curSndDateStr){
|
||||
stop = true
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
|
||||
def curEndStr = DateUtils.add(curStartStr, 1)
|
||||
curStartDateStr = curEndStr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataDealFJW2(curDate, dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
def exportAllBigdataDealFJW(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_DEAL_FJW 导出完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataBuildingFJW(cityCode){
|
||||
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.BUILDALIAS
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.FLOORCNT
|
||||
data << r.UNITCNT
|
||||
data << r.HOUSECNT
|
||||
data << r.IFELEVATOR
|
||||
data << r.ELEVATORCNT
|
||||
data << r.ELEVATORHOUSE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.VALID_FLAG
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def exportEvaluateRate(){
|
||||
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.RATE_TYPE
|
||||
data << r.RATE
|
||||
data << r.RANK
|
||||
data << r.CITY
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "EVALUATE_RATE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
import com.jcraft.jsch.ChannelExec;
|
||||
import com.jcraft.jsch.JSch;
|
||||
import com.jcraft.jsch.Session
|
||||
|
||||
import com.jcraft.jsch.*;
|
||||
|
||||
|
||||
class SSHHelper {
|
||||
|
||||
private Session session = null;
|
||||
private Channel channel = null;
|
||||
|
||||
private String sftpHost = "192.168.0.45";
|
||||
private int sftpPort = 17495;
|
||||
private String sftpUserName = "ccb";
|
||||
private String sftpPassword = "ccb!@#";
|
||||
private int timeout = 30000;
|
||||
/**
|
||||
* 获取连接
|
||||
* @return
|
||||
*/
|
||||
private ChannelExec getChannelExec() {
|
||||
try {
|
||||
if (channel != null && channel.isConnected()) {
|
||||
return (ChannelExec) channel;
|
||||
}
|
||||
JSch jSch = new JSch();
|
||||
if (session == null || !session.isConnected()) {
|
||||
session = jSch.getSession(sftpUserName, sftpHost, sftpPort);
|
||||
session.setPassword(sftpPassword);
|
||||
Properties config = new Properties();
|
||||
config.put("StrictHostKeyChecking", "no");
|
||||
session.setConfig(config);
|
||||
session.setTimeout(timeout);
|
||||
session.connect();
|
||||
}
|
||||
channel = session.openChannel("exec");
|
||||
} catch (Exception e) {
|
||||
if (session != null) {
|
||||
session.disconnect();
|
||||
session = null;
|
||||
}
|
||||
channel = null;
|
||||
}
|
||||
return channel == null ? null : (ChannelExec) channel;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭连接
|
||||
*/
|
||||
private void closeChannel() {
|
||||
try {
|
||||
if (channel != null) {
|
||||
channel.disconnect();
|
||||
channel = null;
|
||||
}
|
||||
if (session != null) {
|
||||
session.disconnect();
|
||||
session = null;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行服务器端命令
|
||||
*/
|
||||
public boolean executeCommand(String command) {
|
||||
boolean flag = false;
|
||||
ChannelExec channelExec = getChannelExec();
|
||||
if (channelExec == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
channelExec.setInputStream(null);
|
||||
channelExec.setErrStream(System.err);
|
||||
channelExec.setCommand(command);
|
||||
|
||||
InputStream ins = channelExec.getInputStream();
|
||||
channelExec.connect();
|
||||
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8"));
|
||||
String line = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
System.out.println(line);
|
||||
}
|
||||
|
||||
reader.close();
|
||||
closeChannel();
|
||||
|
||||
flag = true;
|
||||
} catch (Exception e) {
|
||||
System.out.println(e);
|
||||
flag = false;
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110"));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+102
-3
@@ -66,6 +66,8 @@ class CommonDatautils {
|
||||
public static final DBCollection POI_COLL = MongoUtils.getCollectionByName2("fangjia_base", "baidu_place_poi", mongoDbUrl)
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl)
|
||||
|
||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -204,8 +206,6 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "安居小区");
|
||||
|
||||
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
@@ -218,7 +218,6 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","金辉伯爵山")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -705,6 +704,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","ÎâÖз9ÅªÐ¡Çø")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -828,6 +828,8 @@ class CommonDatautils {
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
@@ -915,6 +917,23 @@ class CommonDatautils {
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static def findDistrictsByCityAndName(String city, String name, String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("cat", cat);
|
||||
q.put("name", name);
|
||||
q.put("visible", 0);
|
||||
|
||||
def list = []
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
list << obj;
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String name, String cat){
|
||||
@@ -1170,6 +1189,46 @@ class CommonDatautils {
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static def findSeaweedByMappingAlias(def city, def region, def name, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
|
||||
BasicDBObject query = new BasicDBObject();
|
||||
query.putAt(QueryOperators.IN, [name]);
|
||||
q.put("mapping_alias", query);
|
||||
|
||||
def obj = SEAWEED_COLL.findOne(q);
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def findSeaweeds(city, cat, name_cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
q.put('name_cat',name_cat)
|
||||
SEAWEED_COLL.count(q)
|
||||
}
|
||||
|
||||
static def findAllSeaweed(def cat , def city){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
def dbcursor = SEAWEED_COLL.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
static def findSeaweedByAddress(def city, def region, def address, def cat){
|
||||
@@ -1315,6 +1374,18 @@ class CommonDatautils {
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
|
||||
public static def getOrgName(def city, def name){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("name", name);
|
||||
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
|
||||
def orgName = null
|
||||
for(DBObject obj : dbCursor){
|
||||
orgName = obj.org_name
|
||||
}
|
||||
return orgName;
|
||||
}
|
||||
|
||||
|
||||
public static def getAllCityRegions(){
|
||||
@@ -3067,5 +3138,33 @@ class CommonDatautils {
|
||||
def count = DEAL_PRICE_COLL.count(q);
|
||||
return count;
|
||||
}
|
||||
|
||||
static def findSqmetreRent(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", weekend)
|
||||
def dbcur = SQMETRE_RENT_COLL.find(q)
|
||||
def dbo = null
|
||||
for(def d : dbcur){
|
||||
dbo = d;
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
static def findNewSqmetreRent(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1)
|
||||
def avg_price = null
|
||||
for(def d : dbcursor){
|
||||
avg_price = d.avg_price;
|
||||
}
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+28
-1
@@ -13,7 +13,34 @@ class DownPdfCheck {
|
||||
|
||||
static main(args) {
|
||||
def task = new DownPdfCheck();
|
||||
task.run();
|
||||
// task.run();
|
||||
|
||||
task.czLogRun()
|
||||
}
|
||||
|
||||
def czLogRun(){
|
||||
def files = getAllFiles("D:\\Êý¾Ý\\³£ÖÝ")
|
||||
if(files){
|
||||
for(def filepath : files){
|
||||
read(filepath);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def read(file){
|
||||
// File file = new File(fileName);
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
if(it.contains("4ead0347b4055be6989e726e")||it.contains("61a2e0")){
|
||||
index++
|
||||
|
||||
println it
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+28
@@ -8,6 +8,7 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jusfoun.JusfounCommon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@@ -79,6 +80,14 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def lastDate = new Date()
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
|
||||
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
// def map2 = [:]
|
||||
// for(def s : list2) {
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
// map2.put(key, row[5])
|
||||
// }
|
||||
|
||||
def result = []
|
||||
for(def rMap : districts){
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
@@ -90,6 +99,25 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << dbo.get("avg_price")
|
||||
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
|
||||
// if(lastMonthPrice && dbo.get("avg_price")){
|
||||
// lastMonthPrice = lastMonthPrice.toInteger()
|
||||
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
|
||||
// if(Math.abs(markup)>10){
|
||||
// def random = 0
|
||||
// if(dbo.get("avg_price")-lastMonthPrice>=0){
|
||||
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
|
||||
// }else{
|
||||
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
|
||||
// }
|
||||
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
|
||||
// m << newPrice.toInteger()
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
/**
|
||||
* 福州数据结果统计
|
||||
*/
|
||||
class FzDataResultStat {
|
||||
|
||||
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
task.regionPriceTrendRun()
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
map1.put(key, row[5])
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row[5])
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
||||
map1.each {k,v->
|
||||
def row = []
|
||||
def keys = k.split("##")
|
||||
def lastPrice = map2.get(k)
|
||||
def markup = null
|
||||
if(v && lastPrice){
|
||||
markup = (v.toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
}
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
def datas1 = []
|
||||
def datas2 = []
|
||||
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
||||
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
||||
def row_num = 0
|
||||
for(def row : rows){
|
||||
if(row_num>0){
|
||||
def region = row[1]
|
||||
if(regions.contains(region)){
|
||||
if(row[5] && Math.abs(row[5])>10){
|
||||
datas1 << row
|
||||
}
|
||||
}else{
|
||||
if(row[5] && Math.abs(row[5])>10){
|
||||
datas2 << row
|
||||
}
|
||||
}
|
||||
}
|
||||
row_num++
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\福州\\6区(超10%)_${curMonth}.xlsx", datas1, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\7县(超10%)_${curMonth}.xlsx", datas2, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def regionPriceTrendRun(){
|
||||
def city = "福州"
|
||||
def add = -6;
|
||||
def months = []
|
||||
while(add<0){
|
||||
months << DateUtils.addMonth(new Date(), add)
|
||||
add++
|
||||
}
|
||||
def prices = exportRegion(city, months)
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
}
|
||||
|
||||
|
||||
def exportRegion(city, months){
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
def datas =[]
|
||||
def header = ["城市", "区域"]
|
||||
for(def m : months){
|
||||
header << m.toString().substring(0, m.toString().lastIndexOf("-"))
|
||||
}
|
||||
datas << header
|
||||
for(def r : regions){
|
||||
def prices = findRegionPrice(city, r, months);
|
||||
datas << prices
|
||||
}
|
||||
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public def findRegionPrice(city, region, weekends){
|
||||
def prices = []
|
||||
prices << city
|
||||
prices << region
|
||||
for(def weekend : weekends){
|
||||
def startDate = DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")
|
||||
def endDate = DateUtils.getLastDayOfMonth(startDate)
|
||||
def avgPrice = queryCityRegionPrice(city, region, startDate, endDate);
|
||||
prices << avgPrice
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
public def queryCityRegionPrice(city, region, startDate, endDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("block", null);
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
|
||||
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public def findWeekends(limit){
|
||||
def index = 1;
|
||||
def weekends = [];
|
||||
def date = new Date();
|
||||
while(index <= limit){
|
||||
def time = DateUtils.getFirstDayOfWeek(date)
|
||||
date = time;
|
||||
weekends << DateUtils.formatDateByPattern(time, "yyyy-MM-dd");
|
||||
index ++;
|
||||
}
|
||||
|
||||
weekends = weekends.reverse();
|
||||
return weekends;
|
||||
}
|
||||
}
|
||||
@@ -48,51 +48,51 @@ import org.bson.types.ObjectId
|
||||
// return location;
|
||||
//}
|
||||
|
||||
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv"))
|
||||
//def map1 = [:]
|
||||
//for(def s : list1){
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
// map1.put(key, row[5])
|
||||
//}
|
||||
//
|
||||
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv"))
|
||||
//def map2 = [:]
|
||||
//for(def s : list2) {
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
// map2.put(key, row[5])
|
||||
//}
|
||||
//def rows = []
|
||||
//map1.each {k,v->
|
||||
// def row = []
|
||||
// def keys = k.split("##")
|
||||
// def lastPrice = map2.get(k)
|
||||
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
||||
def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
||||
if(obj){
|
||||
if(obj.value_type == 1){
|
||||
d["G"] = "指导价"
|
||||
}else if(obj.value_type == 2){
|
||||
d["G"] ="自身样本"
|
||||
}else if(obj.value_type == 3){
|
||||
d["G"] = "第三方"
|
||||
}else if(obj.value_type == 4){
|
||||
d["G"] = "交易数据"
|
||||
}else if(obj.value_type == 5){
|
||||
d["G"] ="周边楼盘"
|
||||
}else if(obj.value_type == 6){
|
||||
d["G"] ="推导"
|
||||
}else if(obj.value_type == 9){
|
||||
d["G"] = "神经网络"
|
||||
}else if(obj.value_type == 10){
|
||||
d["G"] = "无方式"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
map1.put(key, row[5])
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row[5])
|
||||
}
|
||||
def rows = []
|
||||
map1.each {k,v->
|
||||
def row = []
|
||||
def keys = k.split("##")
|
||||
def lastPrice = map2.get(k)
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
||||
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
||||
// if(obj){
|
||||
// if(obj.value_type == 1){
|
||||
// d["G"] = "指导价"
|
||||
// }else if(obj.value_type == 2){
|
||||
// d["G"] ="自身样本"
|
||||
// }else if(obj.value_type == 3){
|
||||
// d["G"] = "第三方"
|
||||
// }else if(obj.value_type == 4){
|
||||
// d["G"] = "交易数据"
|
||||
// }else if(obj.value_type == 5){
|
||||
// d["G"] ="周边楼盘"
|
||||
// }else if(obj.value_type == 6){
|
||||
// d["G"] ="推导"
|
||||
// }else if(obj.value_type == 9){
|
||||
// d["G"] = "神经网络"
|
||||
// }else if(obj.value_type == 10){
|
||||
// d["G"] = "无方式"
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202110.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+5
-2
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
|
||||
retMap.anjuke_esf_price = d.anjuke_esf_price
|
||||
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("58tc_esf_price")
|
||||
}
|
||||
if(!retMap.soufang_esf_price && d.soufang_esf_price){
|
||||
retMap.soufang_esf_price = d.soufang_esf_price
|
||||
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
|
||||
retMap.lianjia_esf_price = d.lianjia_esf_price
|
||||
}
|
||||
if(!retMap.lianjia_esf_price && d.beike_esf_price){
|
||||
retMap.lianjia_esf_price = d.beike_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
|
||||
class ThirdTest {
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new ThirdTest()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.run3()
|
||||
// task.run4()
|
||||
|
||||
}
|
||||
|
||||
def run4(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
if (!d['C']) {
|
||||
// def name = d['A'].replace("(钢混)","").replace("(非钢混)","").replace("(混合)","")
|
||||
// def lists = CommonDatautils.findDistrictsByCityAndName("福州",name, "district")
|
||||
// if(lists){
|
||||
// for(def o : lists){
|
||||
//// if(d['B'] && d['B'].toString().contains(o.address)){
|
||||
// d['C'] = o.region
|
||||
// d['D'] = o.name
|
||||
// d['E'] = "名称去括号中的字"
|
||||
//// }
|
||||
// }
|
||||
// }
|
||||
def rMap = SeaweedDataMatchProcess.doMatch("福州", null, d['A'], d['B']);
|
||||
if(rMap){
|
||||
d['C'] = rMap.region
|
||||
d['D'] = rMap.name
|
||||
d['E'] = "匹配接口"
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run3() {
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
// if(!d['F']){
|
||||
// def price = CommonDatautils.queryDistrictPrice("福州",d['C'], d['D'])
|
||||
// d['F'] = price
|
||||
// }
|
||||
// },0)
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202202.csv"))
|
||||
// def map1 = [:]
|
||||
// for(def s : list1){
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
// map1.put(key, row[5])
|
||||
// }
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv"))
|
||||
def map1 = [:]
|
||||
for(def str : list1){
|
||||
def s = str.split("@\\|@")
|
||||
def key = s[2]+"##"+s[3]+"##"+s[4]
|
||||
def priceMap = [:]
|
||||
map1.put(key, priceMap)
|
||||
|
||||
if(s.size()>=7 && s[6]){
|
||||
priceMap.put("zhugezhaofang", s[6])
|
||||
}
|
||||
|
||||
if(s.size()>=8 && s[7]){
|
||||
priceMap.put("anjuke", s[7])
|
||||
}
|
||||
|
||||
if(s.size()>=9 && s[8]){
|
||||
priceMap.put("soufang", s[8])
|
||||
}
|
||||
|
||||
if(s.size()>=10 && s[9]){
|
||||
priceMap.put("lianjia", s[9])
|
||||
}
|
||||
|
||||
if(s.size()>=11 && s[10]){
|
||||
priceMap.put("cityhouse", s[10])
|
||||
}
|
||||
|
||||
if(s.size()>=12 && s[11]){
|
||||
priceMap.put("fangtan", s[11])
|
||||
}
|
||||
|
||||
if(s.size()>=13 && s[12]){
|
||||
priceMap.put("fangdoudou", s[12])
|
||||
}
|
||||
if(s.size()>=14 && s[13]){
|
||||
priceMap.put("maitian", s[13])
|
||||
}
|
||||
|
||||
try{
|
||||
if(s.size()>=15 && s[14]){
|
||||
priceMap.put("58tc", s[14])
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
// if (!d['F']) {
|
||||
// def lists = CommonDatautils.findDistrictsByCityAndName("福州", d['A'], "district")
|
||||
// if(lists){
|
||||
// for(def o : lists){
|
||||
// if(o.address==d['B']){
|
||||
// d['C'] = o.region
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
def key = "福州##"+d['C']+"##"+d['D']
|
||||
def priceMap = map1.get(key)
|
||||
if(priceMap){
|
||||
if(priceMap.zhugezhaofang){
|
||||
d['F'] = priceMap.zhugezhaofang
|
||||
}
|
||||
if(priceMap.anjuke){
|
||||
d['G'] = priceMap.anjuke
|
||||
}
|
||||
if(priceMap.soufang){
|
||||
d['H'] = priceMap.soufang
|
||||
}
|
||||
if(priceMap.lianjia){
|
||||
d['I'] = priceMap.lianjia
|
||||
}
|
||||
if(priceMap.cityhouse){
|
||||
d['J'] = priceMap.cityhouse
|
||||
}
|
||||
if(priceMap.fangtan){
|
||||
d['K'] = priceMap.fangtan
|
||||
}
|
||||
if(priceMap.fangdoudou){
|
||||
d['L'] = priceMap.fangdoudou
|
||||
}
|
||||
if(priceMap.maitian){
|
||||
d['M'] = priceMap.maitian
|
||||
}
|
||||
|
||||
if(priceMap.get("58tc")){
|
||||
d['N'] = priceMap.get("58tc")
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.modify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供-结果.xlsx",{d->
|
||||
if(d['B'] && !d['D']){
|
||||
if(d['B'].contains("号")){
|
||||
// def price = searchContent2("福州","https://fz.58.com", d['B'], 304)
|
||||
// if(price){
|
||||
// d['C'] = price
|
||||
// d['F'] = "58同城"
|
||||
// }
|
||||
def price = thirdPartyColl("福州", d['B'])
|
||||
if(price){
|
||||
d['D'] = price;
|
||||
d['F'] = "麦田"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def thirdPartyColl(def city, def address){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("address", address)
|
||||
q.put("source","麦田")
|
||||
q.put("s_date",202202)
|
||||
def obj = thirdPartyColl.findOne(q)
|
||||
if(obj){
|
||||
return obj.avg_price
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
//查看此关键字是否有数据
|
||||
def searchContent2(def city,def url,def keyword, def cityid){
|
||||
def newUrl = url+"/esf-ajax/community/pc/autocomplete/?city_id=${cityid}&kw="+URLEncoder.encode(keyword, "utf-8")+"&type=2"
|
||||
def document = CrawalUtil.getDocumentByABuYun(newUrl, "访问过于频繁", "utf-8", null, true);
|
||||
def price = null
|
||||
if(document){
|
||||
println "【"+keyword +"】 查询"
|
||||
def jsonObj = JSONObject.fromObject(document)
|
||||
if(jsonObj && jsonObj.status=="ok"){
|
||||
if(jsonObj.data.size()>0){
|
||||
def datas = jsonObj.data
|
||||
def d = datas[0]
|
||||
price = d.price.toInteger()
|
||||
}
|
||||
}else{
|
||||
println "【"+keyword +"】 无结果"
|
||||
}
|
||||
}
|
||||
return price;
|
||||
}
|
||||
|
||||
def run(){
|
||||
def map = findThirdPrice("福州")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供.xlsx",{d->
|
||||
if(!d['F']){
|
||||
def lists = CommonDatautils.findDistrictsByCityAndName("福州",d['A'], "district")
|
||||
if(!lists){
|
||||
lists = CommonDatautils.findDistrictByAlias("福州",null, d['A'], "district")
|
||||
}
|
||||
for(def o : lists){
|
||||
// if(o.address == d['B']){
|
||||
def priceMap = map.get(o.region+"##"+o.name)
|
||||
d['C'] = priceMap.anjuke_esf_price
|
||||
d['D'] = priceMap.maitian_esf_price
|
||||
d['E'] = priceMap.beike_esf_price
|
||||
d['F'] = o.region
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def findThirdPrice(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202109))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor) {
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.get("lianjia_esf_price") && db.lianjia_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ class EsHouseStat {
|
||||
|
||||
public static void main(String[] args){
|
||||
def task = new EsHouseStat();
|
||||
//task.run();
|
||||
task.run();
|
||||
// task.testRun("ÉîÛÚ")
|
||||
// task.exportOpenLog();
|
||||
// task.exportMongoLog()
|
||||
@@ -59,7 +59,7 @@ class EsHouseStat {
|
||||
// task.szRentHouseStat();
|
||||
// task.addPrice();
|
||||
|
||||
task.statNoHouseCCBData();
|
||||
// task.statNoHouseCCBData();
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
@@ -337,21 +337,6 @@ class EsHouseStat {
|
||||
return total;
|
||||
}
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
def findHouseSource(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
|
||||
@@ -373,6 +358,21 @@ class EsHouseStat {
|
||||
return statMap;
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
def testRun(def city){
|
||||
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ class OpenApiUtils {
|
||||
ret.lng2 = gmap.get("lng")
|
||||
ret.lat2 = gmap.get("lat")
|
||||
ret.estateType = ret.type
|
||||
ret.source = "高德"
|
||||
}catch (Exception e){
|
||||
|
||||
}
|
||||
@@ -250,14 +251,14 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
if(type && type.contains("出入口")){
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
}else{
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
poiList.add(jo);
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
@@ -302,7 +303,7 @@ class OpenApiUtils {
|
||||
def distname = formatName(ret.get("name"))
|
||||
address = ret.get("address")
|
||||
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
|
||||
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
lng2 = location.lng
|
||||
lat2 = location.lat
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
@@ -311,7 +312,7 @@ class OpenApiUtils {
|
||||
def mapRegion = ret.get("area")
|
||||
distRegion = mapRegion
|
||||
districtName = distname
|
||||
if(address && address.contains(mapRegion)){
|
||||
if(address && mapRegion && address.contains(mapRegion)){
|
||||
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
|
||||
}
|
||||
if(ret.get("detail_info")){
|
||||
@@ -330,6 +331,7 @@ class OpenApiUtils {
|
||||
map.name = districtName
|
||||
map.region = distRegion
|
||||
map.estateType = estateType
|
||||
map.source = "百度"
|
||||
rets << map
|
||||
|
||||
lat = null
|
||||
@@ -340,12 +342,18 @@ class OpenApiUtils {
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
// }else{
|
||||
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
// }
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
def score1 = 0
|
||||
if(distname && name){
|
||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
}
|
||||
def score2 = 0
|
||||
if(address && name){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
}
|
||||
if(score1 || score2){
|
||||
rets << map
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
|
||||
|
||||
@@ -31,7 +31,7 @@ class RegionPointCrawl {
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
|
||||
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class SynchSeaweed {
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
|
||||
public static final DBCollection SEAWEED_ONLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed", mongoDbUrl)
|
||||
|
||||
public static final DBCollection SEAWEED_OFFLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed_1215", offlineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new SynchSeaweed();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def datas = CommonDatautils.findAllSeaweed("district", city)
|
||||
saveDatas(datas);
|
||||
}
|
||||
}
|
||||
|
||||
def saveDatas(def datas){
|
||||
def index = 0
|
||||
def ret = []
|
||||
for(def d : datas){
|
||||
index++;
|
||||
ret << d
|
||||
if(ret.size()%10000==0){
|
||||
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
|
||||
SEAWEED_OFFLINE_COLL.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
if(ret){
|
||||
SEAWEED_OFFLINE_COLL.insert(ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.qqmap.TengXunApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
@@ -42,4 +43,29 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"
|
||||
// }
|
||||
//
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
|
||||
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
|
||||
if(d['B']){
|
||||
def json = GaoDeApiUtils.geo("重庆", d['B'])
|
||||
if(json && json.info=="OK"){
|
||||
def datas = json.geocodes
|
||||
if(datas){
|
||||
def row = datas[0]
|
||||
println row
|
||||
d['C'] = row.country
|
||||
d['D'] = row.province
|
||||
d['E'] = row.city.toString().replace("[]","")
|
||||
d['F'] = row.citycode.toString().replace("[]","")
|
||||
d['G'] = row.district.toString().replace("[]","")
|
||||
d['H'] = row.street.toString().replace("[]","")
|
||||
d['I'] = row.number.toString().replace("[]","")
|
||||
d['J'] = row.adcode.toString().replace("[]","")
|
||||
d['K'] = row.location
|
||||
d['L'] = row.level
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
@@ -7,11 +7,11 @@ email.setSSL(true);
|
||||
email.setHostName("smtp.exmail.qq.com");
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
|
||||
email.addTo("gaojuan@fangjia.com"); //接收方
|
||||
email.setFrom("service@fangjia.com"); //发送方
|
||||
email.setSubject("test"); //标题
|
||||
email.setContent("你?", "text/html");
|
||||
email.setContent("测试", "text/html");
|
||||
|
||||
//email.setHostName("smtp.exmail.qq.com");
|
||||
//email.setSSL(true);
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.text.ParseException
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class UpdateStatWeekend {
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", mongoDbUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
static main(args) {
|
||||
def task = new UpdateStatWeekend()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"合肥","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆",
|
||||
def citys = ["苏州"]
|
||||
for(def city : citys){
|
||||
def rets = findAddData(city)
|
||||
process(city, rets)
|
||||
}
|
||||
}
|
||||
|
||||
def process(def city, def rets){
|
||||
|
||||
def index = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def r = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
index.addAndGet(1)
|
||||
updateProcess(r)
|
||||
println index+"/"+rets.size() +" "+r.id
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateProcess(def r){
|
||||
def curDate = DateUtils.parseFromTightYYYYMMDDhhmmss("2021-12-28 11:00:00")
|
||||
if(r.m_date.getTime() < curDate.getTime()){
|
||||
def weekendStr = DateUtils.formatDateByPattern(r.weekend, "yyyy-MM-dd")
|
||||
weekendStr = DateUtils.add(weekendStr, -1)
|
||||
def weekend = DateUtils.parseFromTightYYYYMMDD(weekendStr);
|
||||
weekend = mongoDateTran(weekend)
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("weekend", weekend)
|
||||
data.put("m_date", new Date())
|
||||
|
||||
try{
|
||||
DIST_STAT_COLL.update(new BasicDBObject('_id', r.id), new BasicDBObject('$set', data))
|
||||
}catch(Exception e){
|
||||
DIST_STAT_COLL.remove(new BasicDBObject('_id', r.id))
|
||||
println "[重复] "+ r.id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Date mongoDateTran(Date date) {
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
String dateStr = format.format(date);
|
||||
format.setCalendar(new GregorianCalendar(new SimpleTimeZone(-57600000, "GMT")));
|
||||
try {
|
||||
return format.parse(dateStr);
|
||||
} catch (ParseException e) {
|
||||
return date;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAddData(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-12-27")))
|
||||
def dbcursor = DIST_STAT_COLL.find(q)
|
||||
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << ["id":d._id, "weekend":d.weekend,"c_date":d.c_date,"m_date":d.m_date]
|
||||
if(rets.size()/10000==0){
|
||||
print "."
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+42
-25
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
/**
|
||||
* 模糊地址经纬度更新
|
||||
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
//"北京","常州",
|
||||
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
||||
def citys =["上海"] //CommonDatautils.findAllCity();
|
||||
|
||||
def citys =["北京"] //CommonDatautils.findAllCity();
|
||||
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
||||
//市谊大厦 华三新村 阳光家园
|
||||
for (def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
|
||||
def rets = findVagueAddress(city, districts);
|
||||
findAndUpdateLocation(rets);
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
|
||||
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
|
||||
def findVagueAddress(def city, def districts) {
|
||||
def rets = []
|
||||
def rows = []
|
||||
for (def district : districts) {
|
||||
def address = district.address;
|
||||
if (address) {
|
||||
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
//没地址
|
||||
if (!address) {
|
||||
rets << district;
|
||||
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
if (!findPattern(address)) {
|
||||
rets << district;
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
println district.city + " " + district.address + " 精确地址"
|
||||
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
|
||||
return null;
|
||||
} else {
|
||||
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
|
||||
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
|
||||
matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return s;
|
||||
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
for (def ret : rets) {
|
||||
index++;
|
||||
|
||||
if(ret.name!="药师新村"){
|
||||
continue
|
||||
}
|
||||
|
||||
def locationMap = findLatAndLng(ret);
|
||||
if (locationMap) {
|
||||
updateLocation(locationMap, ret);
|
||||
} else {
|
||||
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
|
||||
}
|
||||
println "【${index}/${rets.size()}】************************************************************************************"
|
||||
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
|
||||
|
||||
}
|
||||
}
|
||||
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
msgMap.db_lng2 = ret.lng2
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
|
||||
if (ret.lat2 != locationMap.lat2) {
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
|
||||
msgMap.m_lat2 = locationMap.lat2
|
||||
msgMap.m_lng2 = locationMap.lng2
|
||||
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
|
||||
msgMap.updateLngFlag = true
|
||||
msgMap.distance = distance
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
}
|
||||
|
||||
if (ret.lat != locationMap.lat) {
|
||||
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
if (udata) {
|
||||
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
|
||||
udata.put("m_date", new Date())
|
||||
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
results << msgMap
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def name, def rets) {
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() > 1) {
|
||||
//相似里找有精确地址
|
||||
for (def mapObj : rets) {
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!map){
|
||||
//交带有交叉口地址
|
||||
for (def mapObj : rets) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
|
||||
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
|
||||
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
def findLatAndLng(ret) {
|
||||
//https://restapi.amap.com/v3/place/text?keywords=
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
|
||||
def map = findStandardObject(ret.city, ret.name, rets);
|
||||
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
//http://api.map.baidu.com/place/v2/search
|
||||
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
if (!map || !map.lat) {
|
||||
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
|
||||
def code = codeMap.get(ret.city);
|
||||
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
|
||||
// if((!map || !map.lat)){
|
||||
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
|
||||
locationMap.id = ret._id
|
||||
locationMap.city = ret.city
|
||||
if (ret.address.toString().endsWith("路")) {
|
||||
if (map.address) {
|
||||
locationMap.address = map.address.replace("(","").replace(")","");
|
||||
}
|
||||
|
||||
|
||||
+6
-5
@@ -37,7 +37,7 @@ class YouDaDataParser {
|
||||
public static final DBCollection youda_newhouse_coll = MongoUtils.getCollectionByName2("business", "res_newdealprice", offlineUrl)
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
//线下:228 线上:45
|
||||
//线下:228 线上:45 测试
|
||||
static def new_targetUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc"
|
||||
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
|
||||
|
||||
@@ -48,7 +48,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-10-01"
|
||||
static def statDate = "2022-02-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//ÓÑ´ïÒ»ÊÖ·¿´æÈëmongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年11月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
@@ -490,8 +490,9 @@ class YouDaDataParser {
|
||||
if(s){
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
if(hits && hits.total instanceof Integer){
|
||||
total = hits.total
|
||||
|
||||
}
|
||||
}
|
||||
return total
|
||||
@@ -565,7 +566,7 @@ class YouDaDataParser {
|
||||
if(obj.premisesName && obj.premisesName!="null"){
|
||||
dbo.premisesName = obj.premisesName
|
||||
}
|
||||
dbo.price = obj.price
|
||||
dbo.price = obj.price.toInteger()
|
||||
dbo.region = obj.region
|
||||
dbo.resId = obj.ydId.toString()
|
||||
dbo.totalPrice = obj.totalPrice
|
||||
|
||||
+8
-5
@@ -62,12 +62,12 @@ class CcbCodeUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
// 高层塔楼 0101 增加“超高层” 0108
|
||||
// 高层板楼 0102 增加“老洋房” 0109
|
||||
// 高层塔楼 0101 增加“超高层” 0109
|
||||
// 高层板楼 0102 增加“老洋房” 0113
|
||||
// 小高层板楼 0103 增加“花园洋房” 0110
|
||||
// 多层板楼 0104 增加“四合院” 0111
|
||||
// 独栋别墅 0105 增加“老式别墅洋房” 0112
|
||||
// 联排别墅 0106
|
||||
// 联排别墅 0106 增加“双拼别墅” 0108
|
||||
// 叠拼别墅 0107
|
||||
// 其他建筑形式 0199
|
||||
|
||||
@@ -84,7 +84,10 @@ class CcbCodeUtils {
|
||||
types << "0110"
|
||||
}
|
||||
if(buildType.contains("老洋房")){
|
||||
types << "0109"
|
||||
types << "0113"
|
||||
}
|
||||
if(buildType.contains("双拼")){
|
||||
types << "0108"
|
||||
}
|
||||
if(buildType.contains("叠拼")){
|
||||
types << "0107"
|
||||
@@ -99,7 +102,7 @@ class CcbCodeUtils {
|
||||
types << "0104"
|
||||
}
|
||||
if(buildType.contains("超高层")){
|
||||
types << "0108"
|
||||
types << "0109"
|
||||
}
|
||||
if(buildType.contains("小高层")){
|
||||
types << "0103"
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
|
||||
class DistrictSelectHouse {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSelectHouse();
|
||||
// task.run()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本V1 -跑地址.xlsx",{d->
|
||||
if(d['A']) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'],d['B'], d['C'], "district")
|
||||
if(dist){
|
||||
d['L'] = dist.address
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本.xlsx",{d->
|
||||
if(d['N']){
|
||||
def floorInfo = d['J']
|
||||
def height = d['N']
|
||||
if(floorInfo && height){
|
||||
height = height.toInteger()
|
||||
def mod = (int)(height/3)
|
||||
def h_start = (height - mod).toInteger()
|
||||
def random_floor = null
|
||||
if(floorInfo.contains("低楼层")){
|
||||
random_floor = random(1, mod)
|
||||
}else if(floorInfo.contains("中楼层")){
|
||||
random_floor = random(mod+1, h_start-1)
|
||||
}else if(floorInfo.contains("高楼层")){
|
||||
random_floor = random(h_start, height)
|
||||
}
|
||||
d['M'] = random_floor
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
|
||||
int random(int i1, int i2) {
|
||||
int max = Math.max(i1, i2);
|
||||
int min = Math.min(i1, i2);
|
||||
return min + RandomUtils.nextInt(max - min + 1);
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "重庆"
|
||||
def weekend = "2022-03-06"
|
||||
def thirdPriceMap = findNearThirdPartyPrice(city)
|
||||
def distStats = findCreditHightDistrict2(city, weekend, thirdPriceMap)
|
||||
findAllHouse(distStats);
|
||||
}
|
||||
|
||||
def findAllHouse(distStats){
|
||||
def houses = []
|
||||
distStats.each{k,v->
|
||||
def arrs = k.split("#")
|
||||
def list = findHouses(arrs[0],arrs[1], arrs[2], 20220201,20220401)
|
||||
if(list){
|
||||
houses.addAll(list)
|
||||
}
|
||||
}
|
||||
if(houses){
|
||||
def rets = []
|
||||
for(def h : houses){
|
||||
def row = []
|
||||
row << h.city
|
||||
row << h.region
|
||||
row << h.district_name
|
||||
row << h.area
|
||||
row << h.avg_price
|
||||
row << h.total_price
|
||||
row << h.direction
|
||||
row << h.estate_type
|
||||
row << h.fitment
|
||||
row << h.floor_info
|
||||
row << h.room
|
||||
row << h.hall
|
||||
row << h.floor
|
||||
row << h.height
|
||||
row << h.p_date
|
||||
rets << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆样本.xlsx", rets, "数据")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def startDate, def endDate){
|
||||
String sql = "select * from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and region='${region}' and district_name='${name}' and p_date >= ${startDate} and p_date<${endDate}"
|
||||
def houses = []
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
houses << data
|
||||
})
|
||||
if(houses.size()>10){
|
||||
houses = houses[0..9]
|
||||
}
|
||||
return houses
|
||||
}
|
||||
|
||||
|
||||
def findNearThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202108))
|
||||
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def thirdPriceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.region+"#"+d.name
|
||||
if(!thirdPriceMap.containsKey(key)){
|
||||
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return thirdPriceMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findCreditHightDistrict2(def city, def weekend, def thirdPriceMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("value_type", 2)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
|
||||
def retMaps = [:]
|
||||
for(def d : dbcursor){
|
||||
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
|
||||
if(thirdPrice) {
|
||||
def precent = (thirdPrice - d.avg_price) * 100 / d.avg_price
|
||||
if (Math.abs(precent) >= 0 && Math.abs(precent) <= 10) {
|
||||
retMaps.put(d.city +"#"+d.region+"#"+d.district_name, d.avg_price)
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMaps
|
||||
}
|
||||
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.xmlbeans.impl.regex.Match
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TempMatchDistrict {
|
||||
static main(args) {
|
||||
def task = new TempMatchDistrict()
|
||||
// task.run()
|
||||
// task.regionConver()
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def total = 0
|
||||
for(def city : citys){
|
||||
def count = CommonDatautils.findAllDistrictCount("district", city)
|
||||
total = total + count
|
||||
println city +" "+count
|
||||
}
|
||||
println "共:"+total
|
||||
}
|
||||
|
||||
def regionConver(){
|
||||
//,"北京"
|
||||
def files = ["广州","杭州","合肥","上海","深圳","石家庄","苏州","天津","武汉","西安","重庆"]
|
||||
for(def f : files){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\均价数据\\"+f+".xlsx",{d->
|
||||
def orgName = CommonDatautils.getOrgName(f, d['M'])
|
||||
if(orgName){
|
||||
d['M'] = orgName
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
|
||||
def files = ["杭州.xlsx"]
|
||||
for(def f : files){
|
||||
def filePath = path+"\\"+f;
|
||||
match(filePath);
|
||||
}
|
||||
}
|
||||
def match(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath,{d->
|
||||
def city = d['B'].replace("市","")
|
||||
// def newRegion = RegionData.formatRegion(city, d['C'])
|
||||
// if(newRegion){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, d['C'], d['D'], null);
|
||||
if(rMap){
|
||||
def dist = CommonDatautils.queryDistrictById(rMap.id)
|
||||
if(dist){
|
||||
d['N'] = dist._id.toString()
|
||||
d['O'] = CommonDatautils.getOrgName(city, dist.region)
|
||||
d['P'] = dist.name
|
||||
d['Q'] = dist.lng2/10000000000.0
|
||||
d['R'] = dist.lat2/10000000000.0
|
||||
d['S'] = getYear(dist.finish_building_date)
|
||||
}
|
||||
}
|
||||
// }
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def getYear(finishBuildingDate){
|
||||
if(finishBuildingDate){
|
||||
Pattern pat3 = Pattern.compile("\\d{4}");
|
||||
def matcher3 = pat3.matcher(finishBuildingDate);
|
||||
if(matcher3.find()){
|
||||
return matcher3.group()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
@@ -38,7 +38,7 @@ class FangjiaClientUtils {
|
||||
}
|
||||
|
||||
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
def content = null;
|
||||
try{
|
||||
|
||||
+1
@@ -29,6 +29,7 @@ class SeaweedDataMatchProcess {
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
rMap.id = obj._id.toString()
|
||||
rMap.address = obj.address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class ZsNoExistsToDB {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new ZsNoExistsToDB()
|
||||
task.run()
|
||||
// task.addDistrictToDB()
|
||||
}
|
||||
|
||||
def addDistrictToDB(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
|
||||
def keys = []
|
||||
def addIndex = 0
|
||||
for(def d : datas){
|
||||
if(d['J'] && d['J']=="新增"){
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def name = d['F'].toString().replace("·","")
|
||||
def address = d['G']
|
||||
def dist = CommonDatautils.findDistrict(city,region,name,"district")
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
|
||||
}
|
||||
if(!dist){
|
||||
keys << city+"##"+region+"##"+name
|
||||
def alias = []
|
||||
def str = d['H']
|
||||
if(str){
|
||||
def arrs = str.split(",")
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address){
|
||||
address = AddressUtil.formatAddress(city, address)
|
||||
}
|
||||
def lng = d['M']
|
||||
def lat = d['L']
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
long lng_new = 0
|
||||
long lat_new = 0
|
||||
long lng2_new = 0
|
||||
long lat2_new = 0
|
||||
if(lng){
|
||||
lng = lng.toString().toDouble()
|
||||
lat = lat.toString().toDouble()
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
lng2 = baidu.x
|
||||
lat2 = baidu.y
|
||||
lng_new = lng *10000000000L
|
||||
lat_new = lat *10000000000L
|
||||
if(lng2){
|
||||
lng2_new = lng2 *10000000000L
|
||||
lat2_new = lat2 *10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", address)
|
||||
if(lng_new){
|
||||
data.put("lat",lat_new)
|
||||
data.put("lng",lng_new)
|
||||
data.put("lat2",lat2_new)
|
||||
data.put("lng2",lng2_new)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(data)
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
|
||||
addIndex++
|
||||
println "[新增] "+addIndex +" "+ data
|
||||
|
||||
}else{
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findOpenLog2()
|
||||
// ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
|
||||
println rets.size()
|
||||
|
||||
// matchMap2()
|
||||
}
|
||||
|
||||
def matchMap2() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
|
||||
if(!d['E']){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
}
|
||||
if(map){
|
||||
println map
|
||||
if(map.adname){
|
||||
d['E'] = map.adname
|
||||
}else if(map.region){
|
||||
d['E'] = map.region
|
||||
}
|
||||
d['F'] = map.name
|
||||
d['G'] = map.address
|
||||
d['K'] = map.estateType
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.lng
|
||||
d['N'] = map.lat2
|
||||
d['O'] = map.lng2
|
||||
d['P'] = map.source
|
||||
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def matchMap(){
|
||||
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
|
||||
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
|
||||
if(!gdData){
|
||||
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
|
||||
}
|
||||
if(gdData){
|
||||
data['E'] = gdData.district
|
||||
data['F'] = gdData.name
|
||||
data['G'] = gdData.address
|
||||
data['H'] = gdData.type
|
||||
data['I'] = gdData.lng
|
||||
data['J'] = gdData.lat
|
||||
data['K'] = gdData.score
|
||||
}
|
||||
|
||||
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
|
||||
if(!bdData){
|
||||
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
|
||||
}
|
||||
if(bdData){
|
||||
data['L'] = bdData.district
|
||||
data['M'] = bdData.name
|
||||
data['N'] = bdData.address
|
||||
data['O'] = bdData.type
|
||||
data['P'] = bdData.lng
|
||||
data['Q'] = bdData.lat
|
||||
data['R'] = bdData.score
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findOpenLog2(){
|
||||
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
|
||||
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def index = 0
|
||||
def count = 0
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d.code == 400){
|
||||
index++
|
||||
def msg = d.msg
|
||||
if(msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def params = d.parameter_map
|
||||
def address = null
|
||||
if(params && params.address){
|
||||
address = params.address
|
||||
}
|
||||
def r = [rMap.city, rMap.region, rMap.name, address]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findOpenLog(){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
|
||||
sqlBuffer.append(" and user_id = '1007569'")
|
||||
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
|
||||
sqlBuffer.append(" and c_date >= '2022-01-01'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def index = 0
|
||||
def count = 0
|
||||
while (rs.next()){
|
||||
def code = rs.getInt("code")
|
||||
if(code == 400){
|
||||
index++
|
||||
def msg = rs.getString("msg")
|
||||
if(msg && msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def params = rs.getObject("parameter_map")
|
||||
println params
|
||||
def r = [rMap.city, rMap.region, rMap.name]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class EsHouseMatchDistrict {
|
||||
|
||||
def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
|
||||
static main(args) {
|
||||
def task = new EsHouseMatchDistrict()
|
||||
task.houseRun()
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
def citys = ["º¼ÖÝ"]//CommonUtils.findAllCity();
|
||||
for(def city : citys){
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-06-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date())
|
||||
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1)
|
||||
def houses = findHouseCounts(city, startDate, endDate);
|
||||
match(city, houses);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def match(city, houses){
|
||||
final CountDownLatch latch = new CountDownLatch(houses.size())
|
||||
final AtomicInteger count = new AtomicInteger(0)
|
||||
final AtomicInteger successCount = new AtomicInteger(0)
|
||||
for(def i=0;i < houses.size();i++){
|
||||
def t = houses[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_seaweed_id){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(t.city, t.region, t.district_name, null);
|
||||
if(rMap){
|
||||
successCount.addAndGet(1)
|
||||
updateHouseMatchDistrictId(t.id, rMap.id)
|
||||
println successCount.get()+" "+t.id+" "+ rMap
|
||||
}
|
||||
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
if(houses){
|
||||
def pre = DataHandleUtils.getDoubleRound(1, successCount.get()*100.0/houses.size());
|
||||
println city +" "+houses.size()+" "+new Date().toString()+" "+pre+"%"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findHouseCounts(def city, def startDate, def endDate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("publish_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("cat", "sell"))
|
||||
.must(date_range).mustNot(QueryBuilders.existsQuery("match_seaweed_id"))
|
||||
// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
// for(def s : ["sell", "rent"]){
|
||||
// inBuilder.should(QueryBuilders.matchQuery("cat", s));
|
||||
// }
|
||||
// queryBuilder.must(inBuilder)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
println houses.size()
|
||||
def rets = []
|
||||
for(def r : houses){
|
||||
if(!r.match_seaweed_id){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def updateHouseMatchDistrictId(def id, def match_id){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.match_seaweed_id='${match_id}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"_id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
println s
|
||||
}
|
||||
|
||||
}
|
||||
+805
@@ -0,0 +1,805 @@
|
||||
package com.fangjia.script.gj.level
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.util.DateUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
/**
|
||||
* 小区潜力指数
|
||||
*/
|
||||
class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictIndexBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||
def code = cityCodeDatas.get(city)
|
||||
def city_avg_total_price = null
|
||||
if(city!="上海"){
|
||||
city_avg_total_price = getAvgTotalPrice(city, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"))
|
||||
if(city_avg_total_price){
|
||||
city_avg_total_price = city_avg_total_price/10000.0
|
||||
city_avg_total_price = DataHandleUtils.getDoubleRound(6, city_avg_total_price)
|
||||
}
|
||||
}else{
|
||||
city_avg_total_price = 531.2615913
|
||||
}
|
||||
if(city_avg_total_price){
|
||||
calcute(city, dists, code, city_avg_total_price)
|
||||
calcuteIndex(city, dists, code, city_avg_total_price)
|
||||
|
||||
}
|
||||
println city +" "+city_avg_total_price
|
||||
}
|
||||
}
|
||||
|
||||
def calcute(city, dists, citycode, city_avg_total_price){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def costDatas = getCostByCity(city)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def datas = new CopyOnWriteArrayList()
|
||||
datas << ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(d)
|
||||
def avg_charge = null
|
||||
def rank = null
|
||||
if (list) {
|
||||
list = sortByDistance(d, list);
|
||||
avg_charge = gtAvgEstateCharge(list)
|
||||
rank = getPriceInRegionRank(d, list, priceMap)
|
||||
}
|
||||
def total_price = getHousePrice(d, dists, priceMap, citycode);
|
||||
def avg_cost = getAvgCost(d, costDatas)
|
||||
|
||||
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
def map = [:]
|
||||
map.id = d._id.toString()
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.name
|
||||
|
||||
map.avg_estate_charge = avg_charge
|
||||
map.price_rank = rank
|
||||
map.avg_cost = avg_cost
|
||||
map.total_price = total_price
|
||||
//map.comp_value = potentiality
|
||||
//map.avg_total_price = total_price_index
|
||||
map.month = "2022-03"
|
||||
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
|
||||
|
||||
map.sell_avg_price = avg_price
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price+""
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await();
|
||||
|
||||
|
||||
// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
|
||||
// def fileName = "/home/gj/potentiality_${city}.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
|
||||
// }
|
||||
// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
|
||||
// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
||||
}
|
||||
|
||||
def calcuteIndex(city, dists, citycode, city_avg_total_price){
|
||||
def detailMap = findAllDistrictDatas(city)
|
||||
def avgMap = calcuteAvgValues(detailMap);
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def avg_total_price = avgMap.avg_price
|
||||
def avg_charge = avgMap.avg_charge
|
||||
def avg_cost = avgMap.avg_cost
|
||||
def avg_rank = avgMap.avg_rank
|
||||
|
||||
def sd_avg_price = avgMap.sd_avg_price
|
||||
def sd_avg_charge = avgMap.sd_avg_charge
|
||||
def sd_avg_cost = avgMap.sd_avg_cost
|
||||
def sd_avg_rank = avgMap.sd_avg_rank
|
||||
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def rMap = detailMap.get(d._id.toString())
|
||||
if(rMap){
|
||||
def total_price_index = 0
|
||||
if (rMap.total_price) {
|
||||
// total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / 313.3693508
|
||||
total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / sd_avg_price
|
||||
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
|
||||
}
|
||||
def avg_charge_index = 0
|
||||
if (rMap.avg_estate_charge) {
|
||||
// avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / 0.772635241
|
||||
avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / sd_avg_charge
|
||||
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
|
||||
}
|
||||
|
||||
def rank_index = 0
|
||||
if (rMap.price_rank) {
|
||||
// rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / 0.261907636
|
||||
rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / sd_avg_rank
|
||||
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
|
||||
}
|
||||
|
||||
def avg_cost_index = 0
|
||||
if (rMap.avg_cost) {
|
||||
// avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / 31.3526959
|
||||
avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / sd_avg_cost
|
||||
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
|
||||
}
|
||||
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
|
||||
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
|
||||
|
||||
|
||||
def year = CommonUtils.getBuildYear(d.finish_building_date)
|
||||
def sqmetre_rent = CommonDatautils.findNewSqmetreRent(d.city,d.region,d.name)
|
||||
def map = [:]
|
||||
map.id = d._id.toString()
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.name
|
||||
|
||||
map.avg_estate_charge = rMap.avg_estate_charge
|
||||
map.price_rank = rMap.price_rank
|
||||
map.avg_cost = rMap.avg_cost
|
||||
map.total_price = rMap.total_price
|
||||
|
||||
map.comp_value = potentiality
|
||||
map.avg_total_price = total_price_index
|
||||
map.month = "2022-03"
|
||||
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
map.finish_date = year
|
||||
map.sell_avg_price = rMap.sell_avg_price
|
||||
if(sqmetre_rent){
|
||||
map.sqmetre_rent = sqmetre_rent
|
||||
}
|
||||
if(rMap.sell_avg_price && sqmetre_rent){
|
||||
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
|
||||
}
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
|
||||
//修改多了会报错
|
||||
// def newUrl = targetUrl +"_update_by_query"
|
||||
// def must = []
|
||||
// def term1 = ["match_phrase":["id":d._id.toString()]]
|
||||
// must << term1
|
||||
// def source = "ctx._source.comp_value="+potentiality+";ctx._source.avg_total_price="+total_price_index+";ctx._source.month='2022-03'"+";ctx._source.m_date='"+DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")+"'"
|
||||
// def params = ["script":["source":source,"lang": "painless"], "query":["bool":["must":must]]]
|
||||
// httpSaveData(newUrl, params);
|
||||
|
||||
println totalCount.get() +" "+d.city+" "+d.region+" "+d.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await();
|
||||
}
|
||||
|
||||
def findAllDistrictDatas(def city){
|
||||
def detailMap = [:]
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
|
||||
for(def rs : houses){
|
||||
def id = rs.id
|
||||
def rMap = detailMap.get(id)
|
||||
if(!rMap){
|
||||
rMap = [:]
|
||||
detailMap.put(id, rMap)
|
||||
}
|
||||
rMap.put("total_price", rs.total_price)
|
||||
rMap.put("price_rank", rs.price_rank)
|
||||
rMap.put("avg_cost", rs.avg_cost)
|
||||
rMap.put("avg_estate_charge", rs.avg_estate_charge)
|
||||
rMap.put("sell_avg_price", rs.sell_avg_price)
|
||||
}
|
||||
return detailMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findDistrictDatasByCity(def city){
|
||||
def must = []
|
||||
def term1 = ["term":["city":city]]
|
||||
must << term1
|
||||
// def params = ["query":["bool":["must":must]]]
|
||||
def params = ["query":["match":["city":city]]]
|
||||
def s = httpSaveData(targetUrl+"/_search", params);
|
||||
def total = 0;
|
||||
if(s){
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
total = hits.total
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def calcuteAvgValues(detailMap){
|
||||
def prices = []
|
||||
def ranks = []
|
||||
def costs = []
|
||||
def charges = []
|
||||
detailMap.each{k,v->
|
||||
if(v.total_price){
|
||||
prices << v.total_price
|
||||
}
|
||||
if(v.price_rank){
|
||||
ranks << v.price_rank
|
||||
}
|
||||
if(v.avg_cost){
|
||||
costs << v.avg_cost
|
||||
}
|
||||
if(v.avg_estate_charge){
|
||||
charges << v.avg_estate_charge
|
||||
}
|
||||
}
|
||||
|
||||
def avg_price = null
|
||||
def dx_avg_price = null
|
||||
def sd_avg_price = null
|
||||
if(prices){
|
||||
def v = prices.sum()/prices.size()
|
||||
avg_price = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
double pow_sum = 0
|
||||
prices.each {p->
|
||||
def sod = Math.abs(p - avg_price)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size())
|
||||
sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price))
|
||||
}
|
||||
def avg_rank = null
|
||||
def dx_avg_rank = null
|
||||
def sd_avg_rank = null
|
||||
if(ranks){
|
||||
def v = ranks.sum()/ranks.size()
|
||||
avg_rank = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
def pow_sum = 0;
|
||||
ranks.each {p->
|
||||
def sod = Math.abs(p - avg_rank)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size())
|
||||
sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank))
|
||||
}
|
||||
|
||||
def avg_cost = null
|
||||
def dx_avg_cost = null
|
||||
def sd_avg_cost = null
|
||||
if(costs){
|
||||
def v = costs.sum()/costs.size()
|
||||
avg_cost = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
double pow_sum = 0;
|
||||
costs.each {p->
|
||||
def sod = Math.abs(p - avg_cost)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size())
|
||||
sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost))
|
||||
}
|
||||
|
||||
|
||||
def avg_charge = null
|
||||
def dx_avg_charge = null
|
||||
def sd_avg_charge = null
|
||||
|
||||
if(charges){
|
||||
def v = charges.sum()/charges.size()
|
||||
avg_charge = DataHandleUtils.getDoubleRound(6, v)
|
||||
|
||||
def pow_sum = 0;
|
||||
charges.each {p->
|
||||
def sod = Math.abs(p - avg_charge)
|
||||
pow_sum = Math.pow(sod, 2) + pow_sum
|
||||
}
|
||||
dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size())
|
||||
sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge))
|
||||
}
|
||||
|
||||
def avgMap = [:]
|
||||
avgMap.put("avg_rank", avg_rank)
|
||||
avgMap.put("avg_price", avg_price)
|
||||
avgMap.put("avg_cost", avg_cost)
|
||||
avgMap.put("avg_charge", avg_charge)
|
||||
|
||||
avgMap.put("dx_avg_rank", dx_avg_rank)
|
||||
avgMap.put("dx_avg_price", dx_avg_price)
|
||||
avgMap.put("dx_avg_cost", dx_avg_cost)
|
||||
avgMap.put("dx_avg_charge", dx_avg_charge)
|
||||
|
||||
avgMap.put("sd_avg_rank", sd_avg_rank)
|
||||
avgMap.put("sd_avg_price", sd_avg_price)
|
||||
avgMap.put("sd_avg_cost", sd_avg_cost)
|
||||
avgMap.put("sd_avg_charge", sd_avg_charge)
|
||||
return avgMap;
|
||||
}
|
||||
|
||||
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1)
|
||||
public double Variance(def x) {
|
||||
int m=x.size();
|
||||
double sum=0;
|
||||
for(int i=0;i<m;i++){//求和
|
||||
sum+=x[i];
|
||||
}
|
||||
double dAve=sum/m;//求平均值
|
||||
double dVar=0;
|
||||
for(int i=0;i<m;i++){//求方差
|
||||
dVar+=(x[i]-dAve)*(x[i]-dAve);
|
||||
}
|
||||
return dVar/m;
|
||||
}
|
||||
|
||||
|
||||
def getAvgTotalPrice(city, startDate, endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def avg_total_price = null
|
||||
while (rs.next()){
|
||||
avg_total_price = rs.getDouble("avg_total_price")
|
||||
}
|
||||
if(avg_total_price){
|
||||
avg_total_price = avg_total_price.toInteger()
|
||||
}
|
||||
return avg_total_price;
|
||||
}
|
||||
|
||||
def getHousePrice(d, dists, priceMap, citycode){
|
||||
//def areas = findHouse(d, 20200101, 20220101)
|
||||
|
||||
// def areas = []
|
||||
// if(citycode){
|
||||
// areas = findHouseByOracle(d._id.toString(), citycode);
|
||||
// }else{
|
||||
// areas = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"))
|
||||
// }
|
||||
def avg_area = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"))
|
||||
if(!avg_area){
|
||||
avg_area = findHousByRegion(d.city, d.region, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"))
|
||||
// if(d.finish_building_date){
|
||||
// def year = CommonUtils.getBuildYear(d.finish_building_date)
|
||||
// if(year){
|
||||
// def startYear = (year.toString().substring(0,3)+"0").toInteger()
|
||||
// def endYear = startYear + 10
|
||||
//
|
||||
// def selectDists = []
|
||||
// for(def ad : dists){
|
||||
// if(d.region == ad.region){
|
||||
// def adYear = CommonUtils.getBuildYear(ad.finish_building_date)
|
||||
// if(adYear){
|
||||
// adYear = adYear.toInteger()
|
||||
// if(adYear>=startYear && adYear<=endYear){
|
||||
// selectDists << ad
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(selectDists){
|
||||
// println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size()
|
||||
// def allAreas = []
|
||||
// def ids = []
|
||||
// if(citycode){
|
||||
// for(def sd : selectDists){
|
||||
// ids << sd._id.toString()
|
||||
// }
|
||||
// if(ids.size()>=901){
|
||||
// ids = ids[0..900]
|
||||
// }
|
||||
//
|
||||
// def idstr = ids.join("','")
|
||||
// allAreas = findHouseByOracle2(idstr,citycode);
|
||||
// }else{
|
||||
// for(def sd : selectDists){
|
||||
// def areaDatas = findHouse(sd, 20190101, 20220101)
|
||||
// allAreas.addAll(areaDatas)
|
||||
// }
|
||||
// }
|
||||
// if(allAreas){
|
||||
// avg_area = allAreas.sum()/allAreas.size();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
if(avg_area){
|
||||
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
if(avg_price){
|
||||
def total_price = (avg_price * avg_area)/10000.0
|
||||
def total = DataHandleUtils.getDoubleRound(2, total_price)
|
||||
return total
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def gtAvgEstateCharge(list){
|
||||
def rets = []
|
||||
for(def i=0;i<list.size();i++){
|
||||
def nb = list[i]
|
||||
def detail = CommonDatautils.findDistrictFieldById(nb._id.toString(), ["estate_charge"])
|
||||
if(detail && detail.estate_charge && detail.estate_charge.toString().isNumber()){
|
||||
rets << detail.estate_charge.toDouble()
|
||||
if(rets.size()>=3){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
def avg_charge = rets.sum()/rets.size();
|
||||
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
|
||||
return avg_charge;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getPriceInRegionRank(d, nbLists, priceMap){
|
||||
if(nbLists){
|
||||
if(nbLists.size()==1){
|
||||
return 0.5;
|
||||
}else{
|
||||
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
def count = 0
|
||||
if(avg_price){
|
||||
for(def nb : nbLists){
|
||||
def price = priceMap.get(nb.city+"##"+nb.region+"##"+nb.name)
|
||||
if(price > avg_price){
|
||||
count ++
|
||||
}
|
||||
}
|
||||
}
|
||||
def pre = (nbLists.size() - count)/nbLists.size()
|
||||
pre = DataHandleUtils.getDoubleRound(2, pre)
|
||||
return pre;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getCostByCity(city){
|
||||
def datas = queryPoi(city)
|
||||
return datas
|
||||
}
|
||||
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
}
|
||||
//排序
|
||||
Collections.sort(list, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
return list;
|
||||
}
|
||||
|
||||
def getAvgCost(d, costDatas){
|
||||
if(costDatas){
|
||||
def costs = []
|
||||
for(def c : costDatas){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, c.lat, c.lng)
|
||||
if(distance <= 1000){
|
||||
costs << c.cost
|
||||
}
|
||||
}
|
||||
if(costs){
|
||||
def avg_cost = costs.sum()/costs.size()
|
||||
avg_cost = DataHandleUtils.getDoubleRound(2, avg_cost)
|
||||
return avg_cost
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def queryPoi(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode,business.keytag as keytag,business.cost as cost FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "市'")
|
||||
sqlBuffer.append(" and business.cost>20 and business.cost<500")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def datas = []
|
||||
def tags = ["法国菜","东南亚菜","日本料理","火锅","海鲜","自助餐","西餐","浙菜","茶餐厅","酒吧","烤肉","潮州菜","云贵菜","韩国料理","中餐","特色火锅","西北菜","串串香","农家乐","烧烤","茶座","上海菜","意大利菜","素菜馆","蛋糕店","东北菜","湘菜","粤菜","咖啡厅","轰趴馆","清真菜","北京菜","川菜","餐饮服务,餐饮相关场所,餐饮相关"]
|
||||
def index = 0
|
||||
while (rs.next()) {
|
||||
index++
|
||||
def tag = rs.getString("keytag")
|
||||
if(tags.contains(tag)){
|
||||
def dMap = [:]
|
||||
dMap.cost = rs.getInt("cost")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
dMap.lng = location.x
|
||||
dMap.lat = location.y
|
||||
}
|
||||
datas << dMap
|
||||
}else{
|
||||
println tag
|
||||
}
|
||||
}
|
||||
println index
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseByOracle(def distId, def citycode){
|
||||
|
||||
def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='${citycode}' AND COMMID='${distId}'";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
allRows << r.BUILDAREA
|
||||
})
|
||||
return allRows
|
||||
}
|
||||
|
||||
def findHouseByOracle2(def idstr, def citycode){
|
||||
|
||||
def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='${citycode}' AND COMMID in ('${idstr}')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
allRows << r.BUILDAREA
|
||||
})
|
||||
return allRows
|
||||
}
|
||||
|
||||
def findHouse(def dist, def startDate, def endDate){
|
||||
def nameQuery = getAllName(dist);
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+dist.city+"'")
|
||||
sqlBuffer.append(" and region = '"+dist.region+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
sqlBuffer.append(" and (" + nameQuery + ")");
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def areas = []
|
||||
def avg_area = null
|
||||
while (rs.next()){
|
||||
// areas << rs.getDouble("area")
|
||||
avg_area = rs.getDouble("avg_area")
|
||||
}
|
||||
if(avg_area){
|
||||
avg_area = DataHandleUtils.getDoubleRound(2, avg_area)
|
||||
}
|
||||
return avg_area;
|
||||
}
|
||||
|
||||
|
||||
def findHousByRegion(def city, def region, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and region = '"+region+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def avg_area = null
|
||||
while (rs.next()){
|
||||
avg_area = rs.getDouble("avg_area")
|
||||
}
|
||||
if(avg_area){
|
||||
avg_area = DataHandleUtils.getDoubleRound(2, avg_area)
|
||||
}
|
||||
return avg_area;
|
||||
}
|
||||
|
||||
def getAllName(dist){
|
||||
Set<String> aliasNames = new LinkedHashSet<String>();
|
||||
aliasNames.add(dist.name);
|
||||
if (dist != null && dist.alias != null) {
|
||||
aliasNames.addAll(dist.alias);
|
||||
}
|
||||
|
||||
List<String> namePars = new ArrayList<String>();
|
||||
if(aliasNames!=null && aliasNames.size()>0) {
|
||||
for(String name : aliasNames){
|
||||
if(name && name.trim().length() > 1){
|
||||
namePars.add("district_name ='" + name + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(namePars.size() > 0){
|
||||
return namePars.join(" OR ")
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// def updateHouseMatchDistrictId(def id, def match_id){
|
||||
// def header = [:]
|
||||
// header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
|
||||
// String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
// def body = """
|
||||
// {
|
||||
// "script": {
|
||||
// "source": "ctx._source.match_seaweed_id='${match_id}'",
|
||||
// "lang": "painless"
|
||||
// },
|
||||
// "query": {
|
||||
// "term": {
|
||||
// "_id": "${id}"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// """
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// println s
|
||||
// }
|
||||
|
||||
|
||||
|
||||
def httpSaveData(def newUrl,def params){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
|
||||
doPost(newUrl,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true);
|
||||
conn.setUseCaches(false);
|
||||
conn.setConnectTimeout(5000);
|
||||
conn.setReadTimeout(5000);
|
||||
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
conn.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
} else {
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||
param = JacksonUtil.beanToJson(params)
|
||||
}
|
||||
|
||||
conn.connect();
|
||||
|
||||
//println param.toString()
|
||||
//输出流
|
||||
if (StringUtils.isNotEmpty(param)) {
|
||||
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||
mWriter.write(param);
|
||||
mWriter.flush();
|
||||
mWriter.close();
|
||||
}
|
||||
if (conn.getHeaderField("Content-Encoding") != null
|
||||
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||
} else {
|
||||
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||
}
|
||||
char[] bf = new char[1024];
|
||||
int len;
|
||||
while ((len = inStream.read(bf)) != -1) {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class DsElasticUtils {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+210
-7
@@ -1,11 +1,20 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.codehaus.jackson.sym.NameN
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
class DistrictLocationUpdate {
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
@@ -19,20 +28,214 @@ class DistrictLocationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationUpdate()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.mapMatch()
|
||||
// task.updateLocationToDB()
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
|
||||
|
||||
|
||||
updateLocation(city, dists);
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"])
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"]
|
||||
for(def d : dists){
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(d.lng){
|
||||
lng = d.lng/10000000000.0
|
||||
lat = d.lat/10000000000.0
|
||||
}
|
||||
rets << [d.city, d.region, d.name, d.address, lng,lat]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
|
||||
if(!d['H'] && d['C']=="华侨城龙居御景二期"){
|
||||
index++
|
||||
// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德")
|
||||
// println d['A']+" "+d['B']+" "+d['C']
|
||||
// if(gdData){
|
||||
// d['G'] = gdData.district
|
||||
// d['H'] = gdData.name
|
||||
// d['I'] = gdData.address
|
||||
// d['J'] = gdData.type
|
||||
// d['K'] = gdData.lng
|
||||
// d['L'] = gdData.lat
|
||||
// d['M'] = gdData.score
|
||||
// if(d['E'] && d['K']){
|
||||
// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
|
||||
// d['N'] = distance
|
||||
// }
|
||||
// }
|
||||
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
}
|
||||
if(map){
|
||||
if(map.adname){
|
||||
d['G'] = map.adname
|
||||
}else if(map.region){
|
||||
d['G'] = map.region
|
||||
}
|
||||
d['H'] = map.name
|
||||
d['I'] = map.address
|
||||
if(map.type){
|
||||
d['J'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['J'] = map.estateType
|
||||
}
|
||||
d['K'] = map.lng
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.score
|
||||
if(d['E'] && d['K']){
|
||||
def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
|
||||
d['N'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateLocationToDB(){
|
||||
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
|
||||
if(d['N'] && d['N']>1000){
|
||||
if(d['M']>=0.4 && d['K']){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d['A'])
|
||||
query.put("region", d['B'])
|
||||
query.put("name", d['C'])
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
|
||||
Long lng0 = d['K'].toString().toDouble()*10000000000L
|
||||
Long lat0 = d['L'].toString().toDouble()*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble())
|
||||
Long lng2 = gmap.get("lng")*10000000000L
|
||||
Long lat2 = gmap.get("lat")*10000000000L
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("lng2", lng2)
|
||||
DbUtils.seaweed.update(query, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
|
||||
continue;
|
||||
}
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def queryPoiMap(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
|
||||
+2
-2
@@ -40,9 +40,9 @@ class LocationTest {
|
||||
// System.out.println(str);
|
||||
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.gj.paimai.ChromeDriverPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
public class BeiKeCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
keys << ["city":d.city, "name":d.district_name]
|
||||
}
|
||||
for(def m : keys){
|
||||
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
|
||||
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
|
||||
if(list && list.size()==1){
|
||||
def dist = list[0]
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", m.city)
|
||||
query.put("region", null)
|
||||
query.put("district_name", m.name)
|
||||
// query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def getPageUrls(driver, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findRegionByThirdPary(city, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", district_name)
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
|
||||
def dbo = null;
|
||||
for(def d : dbcursor){
|
||||
dbo = d
|
||||
}
|
||||
if(dbo){
|
||||
return dbo.region
|
||||
}
|
||||
return null;
|
||||
}
|
||||
def getContent(driver, detailUrl){
|
||||
driver.get(detailUrl)
|
||||
def str = driver.getPageSource()
|
||||
if(str && str.contains("<title>登录</title>")){
|
||||
getLogin(driver)
|
||||
return getContent(driver, detailUrl)
|
||||
}
|
||||
return str;
|
||||
}
|
||||
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
|
||||
if(detailUrls){
|
||||
for(def i=0;i<detailUrls.size();i++){
|
||||
def detailUrl = detailUrls[i]
|
||||
try{
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
|
||||
def content = getContent(driver, detailUrl)
|
||||
def doc = Jsoup.parse(content);
|
||||
def eles = JQueryUtils.select(".deal-bread a", doc)
|
||||
def detail = new BasicDBObject()
|
||||
if(eles && eles.size()>=4){
|
||||
detail.city = eles[0].text().replace("房产网","")
|
||||
detail.region = eles[2].text().replace("二手房成交","")
|
||||
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
|
||||
}
|
||||
if(detail.district_name) {
|
||||
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def getAllCityUrls(driver){
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(!e.text().contains("南京")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
public class ChromeDriverPool {
|
||||
|
||||
private static ConcurrentHashMap<Integer,Object> pool ;//线程池
|
||||
private static int coreSize = 2;//核心线程池,一直驻留的资源
|
||||
private static int maxSize = 10;//TODO 设置最大数量资源,用于解决瞬时并发,瞬时并发过后要定时去识别空闲的释放掉,保留coreSize的资源
|
||||
private static AtomicInteger count = new AtomicInteger(0);//原子自增!!
|
||||
|
||||
static {
|
||||
//驱动初始化
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
pool = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
private ChromeDriverPool(){
|
||||
}
|
||||
|
||||
public static void init(int coreSize){//线程池数量初始化
|
||||
ChromeDriverPool.coreSize = coreSize;
|
||||
}
|
||||
|
||||
public static Worker getPool(){
|
||||
if(count.getAndIncrement() < coreSize){//自增!!!
|
||||
return push(new Worker());
|
||||
}else{
|
||||
return pop();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Worker push(Worker worker){
|
||||
pool.put(count.get(),worker);
|
||||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* 循环获取未使用的资源
|
||||
* @return
|
||||
*/
|
||||
public static Worker pop(){
|
||||
Set<Map.Entry<Integer, Object>> sets = pool.entrySet();
|
||||
Iterator<Map.Entry<Integer, Object>> its = sets.iterator();
|
||||
while(its.hasNext()){
|
||||
Map.Entry<Integer, Object> workEntry = its.next();
|
||||
Worker work = (Worker)workEntry.getValue();
|
||||
if(work.status == 1){
|
||||
return work;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-1
@@ -514,8 +514,9 @@ class GaoDeUtils {
|
||||
// "2a1fb75990f2b39f6221e50892dd3988"
|
||||
// ]
|
||||
|
||||
//35925efd7333f1df52145e51b8e4e56a 15021384995
|
||||
def KeyList = [
|
||||
"10bcb421d2ee4e51ed9268d6dca58ad1"
|
||||
"10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975
|
||||
]
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
|
||||
+459
@@ -0,0 +1,459 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.gj.paimai.ChromeDriverPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
public class LianjiaCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityUrls = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
|
||||
for(def cityUrl : cityUrls){
|
||||
def detailUrls = getPageUrls(driver, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
poolCrawl( detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def poolCrawl(detailUrls){
|
||||
//模拟并发
|
||||
int parallelSize = 10;
|
||||
final CountDownLatch latch = new CountDownLatch(1);
|
||||
ExecutorService pool = Executors.newFixedThreadPool(parallelSize);
|
||||
|
||||
for (int i = 0; i < detailUrls.size(); i++) {
|
||||
final def detailUrl = detailUrls[i]
|
||||
pool.execute(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
Worker work = null;
|
||||
try {
|
||||
latch.await();//阻塞所有线程
|
||||
System.out.println(Thread.currentThread().getName());
|
||||
while (true) {//业务需求,一直等待直到获取到驱动
|
||||
work = ChromeDriverPool.getPool();
|
||||
if (work != null) {
|
||||
break;
|
||||
}
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
work.start();
|
||||
ChromeDriver driver = work.getDriver();
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println detail.city + " " + detail.region + " " + detail.district_name
|
||||
System.out.println(Thread.currentThread().getName()+":"+driver);
|
||||
Thread.sleep(1000 * 10);
|
||||
} catch (Exception e) {
|
||||
System.out.println(e.getMessage());
|
||||
}finally {
|
||||
work.shutdown();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
if(d.region){
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if(newRegion && newRegion!=d.region){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d.city)
|
||||
query.put("region", d.region)
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", newRegion)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
keys << ["city":d.city, "name":d.district_name]
|
||||
}
|
||||
for(def m : keys){
|
||||
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
|
||||
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
|
||||
if(list && list.size()==1){
|
||||
def dist = list[0]
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", m.city)
|
||||
query.put("region", null)
|
||||
query.put("district_name", m.name)
|
||||
// query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
println m.city+ " "+m.name +" "+dist.region
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def getPageUrls(driver, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findRegionByThirdPary(city, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", district_name)
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
|
||||
def dbo = null;
|
||||
for(def d : dbcursor){
|
||||
dbo = d
|
||||
}
|
||||
if(dbo){
|
||||
return dbo.region
|
||||
}
|
||||
return null;
|
||||
}
|
||||
def getContent(driver, detailUrl){
|
||||
driver.get(detailUrl)
|
||||
def str = driver.getPageSource()
|
||||
if(str && str.contains("<title>登录</title>")){
|
||||
getLogin(driver)
|
||||
return getContent(driver, detailUrl)
|
||||
}
|
||||
return str;
|
||||
}
|
||||
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
|
||||
if(detailUrls){
|
||||
for(def i=0;i<detailUrls.size();i++){
|
||||
def detailUrl = detailUrls[i]
|
||||
try{
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
|
||||
def content = getContent(driver, detailUrl)
|
||||
def doc = Jsoup.parse(content);
|
||||
def eles = JQueryUtils.select(".deal-bread a", doc)
|
||||
def detail = new BasicDBObject()
|
||||
if(eles && eles.size()>=4){
|
||||
detail.city = eles[0].text().replace("房产网","")
|
||||
detail.region = eles[2].text().replace("二手房成交","")
|
||||
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
|
||||
}
|
||||
if(detail.district_name) {
|
||||
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def getAllCityUrls(driver){
|
||||
driver.get("https://www.lianjia.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(!href.contains("fang.lianjia") && e.text().contains("福州")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class Worker {
|
||||
private ChromeDriver driver;
|
||||
/**
|
||||
* 状态修改要让其他线程可见
|
||||
* 1就绪 2运行
|
||||
*/
|
||||
private volatile int status;
|
||||
|
||||
public Worker(){
|
||||
driver = new ChromeDriver();
|
||||
status = 1;
|
||||
}
|
||||
|
||||
|
||||
public void shutdown(){
|
||||
this.status = 1;
|
||||
}
|
||||
|
||||
|
||||
public void start(){
|
||||
this.status = 2;
|
||||
}
|
||||
|
||||
public ChromeDriver getDriver() {
|
||||
return driver;
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.fangjia.script.gj.price
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
class FzMatchTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FzMatchTest();
|
||||
// task.run()
|
||||
task.caluatePPE()
|
||||
}
|
||||
|
||||
def caluatePPE(){
|
||||
def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区_结果1).xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { data ->
|
||||
if(data['F'] && data['G']){
|
||||
def dist = CommonDatautils.findDistrict("福州", data['F'], data['G'], "district")
|
||||
if(dist){
|
||||
def avg_price = findAvgPrice(dist._id.toString())
|
||||
if(data['i'] && avg_price){
|
||||
def weekendPrice = data['i'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
data['J'] = ppe15
|
||||
}
|
||||
}
|
||||
}
|
||||
},1)
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区).xlsx"
|
||||
def city = "福州"
|
||||
ExcelUtils.multiThreadingModify(filePath, { data ->
|
||||
// if(data['G']){
|
||||
// if(!data['H']){
|
||||
// def dist = CommonDatautils.findDistrict(city, data['F'], data['G'], "district")
|
||||
//
|
||||
// if(dist && dist.finish_building_date){
|
||||
// data['H'] = dist.finish_building_date.substring(0,4)
|
||||
// }
|
||||
// }
|
||||
// if(!data['I']){
|
||||
// def price = CommonDatautils.queryDistrictPrice(city, data['F'], data['G'])
|
||||
// if(price){
|
||||
// data['I'] = price
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
if(!data['G'] && data['B']){
|
||||
def name = data['B'].replace("(", "").replace(")","").replace("\\","").replace("#","")
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, null, name, null);
|
||||
if(rMap) {
|
||||
data['F'] = rMap.region
|
||||
data['G'] = rMap.name
|
||||
def dist = CommonDatautils.findDistrictById(rMap.id)
|
||||
if(dist && dist.finish_building_date){
|
||||
data['H'] = dist.finish_building_date.substring(0,4)
|
||||
}
|
||||
|
||||
def price = CommonDatautils.queryDistrictPrice(city, rMap.region, rMap.name)
|
||||
if(price){
|
||||
data['I'] = price
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 1);
|
||||
}
|
||||
}
|
||||
+62
-24
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
def source = "58同城"
|
||||
def root = "58.com"
|
||||
def shortCityMap = findCitySeeds(root)
|
||||
def datas = findCrawlData(source, shortCityMap, root)
|
||||
def list = []
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def findCitySeeds(root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("root", root)
|
||||
q.put("city", "福州")
|
||||
|
||||
def dbcursor = districtseedsColl.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(!cityMap.keySet().contains(d.city)){
|
||||
cityMap.put(d.city, d.city_short)
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
}
|
||||
|
||||
def findCrawlData(def source, shortCityMap, def root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","¸£ÖÝ")
|
||||
q.put("source", "³ÇÊз¿²ú")
|
||||
q.put("s_date",202012)
|
||||
q.put("source", source)
|
||||
q.put("s_date",202112)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = ".cityhouse.cn/ha/"
|
||||
def prefix = "xiaoqu/"
|
||||
def keyList =new HashSet()
|
||||
def index = 0
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
@@ -76,33 +96,42 @@ class CrawlDistrictIdSelect {
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
|
||||
|
||||
if(url){
|
||||
def district_id = ""
|
||||
if(url.contains(prefix)){
|
||||
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
|
||||
}else{
|
||||
district_id = url;
|
||||
}
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
def ids = retMap.get(db.city)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
retMap.put(db.city, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "cityhouse.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
if(!keyList.contains(db.city+"#"+district_id)){
|
||||
def count = findIsExists(db.city, district_id, root)
|
||||
if(!count){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "58.com"
|
||||
map.city_short= shortCityMap.get(db.city)
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
map.name = db.name
|
||||
ids << map
|
||||
}else{
|
||||
index++
|
||||
println "[${index}]已存在:"+db.city +" "+db.name
|
||||
}
|
||||
keyList << db.city+"#"+district_id
|
||||
}else{
|
||||
println "ÖØ¸´£º"+key
|
||||
println "重复:"+db.city+"#"+db.name
|
||||
}
|
||||
}else{
|
||||
println url +" Á´½Ó²»ºÏÒªÇó"
|
||||
@@ -110,4 +139,13 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findIsExists(city, district_id, root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("district_id", district_id)
|
||||
q.put("root",root)
|
||||
def count = districtseedsColl.count(q)
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -68,7 +68,7 @@ class FZloationUpdate {
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ class FZloationUpdate {
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
if(reps && reps.count && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -29,25 +30,33 @@ import java.text.SimpleDateFormat
|
||||
// }
|
||||
//})
|
||||
|
||||
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
//def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
//
|
||||
//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
|
||||
//def countMap = [:]
|
||||
//onLineSqlInstance.eachRow(sql,{r->
|
||||
// def addTime = r.visit_time
|
||||
// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
|
||||
// def ips = countMap.get(month)
|
||||
// if(!ips){
|
||||
// ips = new HashSet();
|
||||
// countMap.put(month, ips)
|
||||
// }
|
||||
// ips << r.ip
|
||||
//});
|
||||
//
|
||||
//countMap.each {k,v->
|
||||
// println k+" "+v.size()
|
||||
//}
|
||||
|
||||
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
|
||||
def countMap = [:]
|
||||
onLineSqlInstance.eachRow(sql,{r->
|
||||
def addTime = r.visit_time
|
||||
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
|
||||
def ips = countMap.get(month)
|
||||
if(!ips){
|
||||
ips = new HashSet();
|
||||
countMap.put(month, ips)
|
||||
}
|
||||
ips << r.ip
|
||||
});
|
||||
|
||||
countMap.each {k,v->
|
||||
println k+" "+v.size()
|
||||
def dists = CommonDatautils.findAllDistrictByCity("ÉϺ£", "district", ["city","region","name","finish_building_date"])
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
def year = CommonUtils.getDistrictAge(d)
|
||||
rets << [d.city,d.region, d.name, year]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\Êý¾Ý\\ͳ¼Æ·¿Áä.xlsx", rets, "Êý¾Ý")
|
||||
|
||||
|
||||
|
||||
|
||||
+50
-1
@@ -42,6 +42,55 @@ class SFTP {
|
||||
return sftp;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 删除本地文件
|
||||
*
|
||||
* @param filePath
|
||||
* @return
|
||||
*/
|
||||
public boolean deleteFile(String filePath) {
|
||||
File file = new File(filePath);
|
||||
if (!file.exists()) {
|
||||
return false;
|
||||
}
|
||||
if (!file.isFile()) {
|
||||
file.deleteDir();
|
||||
return false;
|
||||
}
|
||||
boolean rs = file.delete();
|
||||
return rs;
|
||||
}
|
||||
|
||||
public static void deleteSFTP(String directory, ChannelSftp sftp) {
|
||||
try {
|
||||
if (isDirExist(directory, sftp)) {
|
||||
Vector<ChannelSftp.LsEntry> vector = sftp.ls(directory);
|
||||
if (vector.size() == 1) { // 文件,直接删除
|
||||
sftp.rm(directory);
|
||||
} else if (vector.size() == 2) { // 空文件夹,直接删除
|
||||
sftp.rmdir(directory);
|
||||
} else {
|
||||
String fileName = "";
|
||||
// 删除文件夹下所有文件
|
||||
for (ChannelSftp.LsEntry en : vector) {
|
||||
fileName = en.getFilename();
|
||||
if (".".equals(fileName) || "..".equals(fileName)) {
|
||||
continue;
|
||||
} else {
|
||||
deleteSFTP(directory + "/" + fileName);
|
||||
}
|
||||
}
|
||||
// 删除文件夹
|
||||
sftp.rmdir(directory);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传文件
|
||||
* @param directory 上传的目录
|
||||
@@ -132,7 +181,7 @@ class SFTP {
|
||||
sftp.cd(filePath.toString());
|
||||
} else {
|
||||
// 建立目录
|
||||
sftp.mkdir(filePath.toString());
|
||||
sftp.mkdir(filePath.toString())
|
||||
// 进入并设置为当前目录
|
||||
sftp.cd(filePath.toString());
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");//·¢ËÍ·½
|
||||
email.setSubject(mail.subject); //±êÌâ
|
||||
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");
|
||||
email.setSubject(mail.subject);
|
||||
|
||||
+3
@@ -120,6 +120,9 @@ class AddressUtil {
|
||||
if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')) {
|
||||
return formatAddress
|
||||
}
|
||||
if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷)')) {
|
||||
return formatAddress
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.interactions.Actions
|
||||
import org.openqa.selenium.support.ui.ExpectedCondition
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
class HouseBookChromeCrawl {
|
||||
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
static def city = "重庆"
|
||||
static main(args) {
|
||||
def task = new HouseBookChromeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def listPageUrl = getIndexPage(driver)
|
||||
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
|
||||
keywords.remove("城南家园")
|
||||
keywords.remove("康居西城")
|
||||
|
||||
println city +"共 "+keywords.size()+" 个小区"
|
||||
getDistrictIndex(driver, keywords)
|
||||
|
||||
}
|
||||
|
||||
def getListPage(driver, url){
|
||||
driver.get(url)
|
||||
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource())
|
||||
def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc)
|
||||
if(total){
|
||||
def page = getTotalPage(total, 6)
|
||||
def keywords = getAllDist(driver, url, page);
|
||||
return keywords
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getDistrictIndex(driver, keywords){
|
||||
for(def word : keywords){
|
||||
def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8")
|
||||
goToDetailIndex(driver, code)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getAllDist(driver, url, page){
|
||||
def keywords = []
|
||||
for(def i=1;i<=page;i++){
|
||||
driver.get(url+"?selectPages="+i)
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def eles = JQueryUtils.select(".title-area .title-text", doc)
|
||||
for(def e : eles){
|
||||
def text = e.text()
|
||||
if(text=="富州北郡"){
|
||||
println i
|
||||
}
|
||||
keywords << text
|
||||
}
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
def getTotalPage(def total, def pageSize){
|
||||
def mode = total.toInteger()/pageSize
|
||||
if(mode > mode.toInteger()){
|
||||
mode = mode.toInteger() +1
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
|
||||
|
||||
def goToDetailIndex(driver, code){
|
||||
def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind="
|
||||
driver.get(url)
|
||||
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main"))));
|
||||
def docment = Jsoup.parse(driver.getPageSource());
|
||||
def dist_name = JQueryUtils.select(".info-pname", docment).text()
|
||||
def addInfo = JQueryUtils.select(".addr-info", docment).text()
|
||||
def region = null
|
||||
if(addInfo.contains(":") && addInfo.contains("-")){
|
||||
region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-"))
|
||||
}
|
||||
|
||||
def address = JQueryUtils.select(".addr-info span", docment).text()
|
||||
|
||||
// wait.until(new ExpectedCondition<WebElement>(){
|
||||
// @Override
|
||||
// public WebElement apply(WebDriver d){
|
||||
// return d.findElement(By.linkText("[更多]"))
|
||||
// }
|
||||
// }).click()
|
||||
driver.findElement(By.linkText("[更多]")).click()
|
||||
// Thread.sleep(10)
|
||||
// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl"))));
|
||||
def lis = checkElement(driver, "#shouUl .dongli");
|
||||
|
||||
def datas = []
|
||||
if(lis){
|
||||
for(def li : lis){
|
||||
def id = li.attr("id")
|
||||
def houseNum = li.select(".title-span").text()
|
||||
def map = [:]
|
||||
map.id = id
|
||||
map.city = city
|
||||
map.region = region
|
||||
map.district_name = dist_name
|
||||
map.address = address
|
||||
map.house_num = houseNum
|
||||
map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","")
|
||||
map.type = li.select(".dong_ul2 .title-span-hbType").text()
|
||||
def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","")
|
||||
if(floor_heigth){
|
||||
map.floor_heigth = floor_heigth
|
||||
}
|
||||
def room_count = li.select(".dong_ul2 label:contains(总户数:)").text()
|
||||
if(room_count){
|
||||
room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户"))
|
||||
if(room_count){
|
||||
map.household_count = room_count
|
||||
}
|
||||
}
|
||||
datas << map
|
||||
}
|
||||
}
|
||||
|
||||
println dist_name +" 共 "+datas.size()+" 栋"
|
||||
for(def m : datas){
|
||||
try{
|
||||
driver.findElement(By.id(m.id)).click()
|
||||
|
||||
def rows = parseDetail(driver)
|
||||
println "栋divId: "+m.id +" 室:"+rows.size()
|
||||
saveDate(m, rows)
|
||||
}catch(Exception e){
|
||||
println dist_name+" 栋divId: "+m.id+" 找不到相应的div element"
|
||||
}
|
||||
|
||||
// driver.navigate().back();
|
||||
}
|
||||
}
|
||||
|
||||
def checkElement(driver, className){
|
||||
def loop = 0
|
||||
def stop = false
|
||||
def lis = null
|
||||
while (!stop){
|
||||
loop++
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
lis = JQueryUtils.select(className, doc)
|
||||
if(lis || loop>=10){
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}
|
||||
return lis
|
||||
|
||||
}
|
||||
|
||||
def parseDetail(driver){
|
||||
def trs = checkElement(driver, ".rooms tr");
|
||||
def rows = []
|
||||
for(def tr : trs){
|
||||
def tds = tr.select("td")
|
||||
if(tds){
|
||||
def floor = null
|
||||
for(def j=0;j<tds.size();j++){
|
||||
if(j == 0){
|
||||
floor = tds[0].text().replace("第","").replace("层","")
|
||||
}else {
|
||||
def td = tds[j]
|
||||
def detail = [:]
|
||||
def room_num = td.select("b").text()
|
||||
if (room_num) {
|
||||
detail.room_num = room_num
|
||||
}
|
||||
def area = td.select("span:contains(面积:)").text().replace("面积:", "").replace("㎡", "")
|
||||
if (area) {
|
||||
detail.area = area.toDouble()
|
||||
}
|
||||
def avg_price = td.select("span:contains(单价:)").text().replace("单价:", "").replace("元/㎡", "")
|
||||
if (avg_price) {
|
||||
detail.avg_price = avg_price.toInteger()
|
||||
}
|
||||
|
||||
def house_type = td.select("span:contains(房屋类型:)").text().replace("房屋类型:", "")
|
||||
if(house_type){
|
||||
detail.estate_type = house_type
|
||||
}
|
||||
if(detail){
|
||||
=-09ql.floor = floor
|
||||
rows << detail
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}+*-*
|
||||
|
||||
|
||||
def getIndexPage(driver){
|
||||
driver.get("http://www.house-book.com.cn/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select("a:contains(新 房)", cityDoc)
|
||||
if(cityEles){
|
||||
def listPageUrl = cityEles[0].attr("href")
|
||||
return listPageUrl
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def saveDate(m, rows){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", m.region)
|
||||
q.put("district_name", m.district_name)
|
||||
def datas = []
|
||||
for(def map : rows){
|
||||
q.put("house_num", m.house_num)
|
||||
q.put("room_num", map.room_num)
|
||||
q.put("floor_heigth" , m.floor_heigth)
|
||||
q.put("floor", map.floor)
|
||||
q.put("area", map.area)
|
||||
def obj = loudongConn.findOne(q)
|
||||
if(!obj){
|
||||
def data = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
data.put(k,v)
|
||||
}
|
||||
map.each{k,v->
|
||||
data.put(k,v)
|
||||
}
|
||||
data.put("c_date", new Date())
|
||||
datas << data
|
||||
}
|
||||
}
|
||||
if(datas){
|
||||
loudongConn.insert(datas)
|
||||
}
|
||||
}
|
||||
}
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.alibaba.excel.EasyExcel
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.*
|
||||
|
||||
/**
|
||||
* 成交详情导出
|
||||
*
|
||||
* @author shenjian* @since 2021/3/29
|
||||
*/
|
||||
class DealPriceDetailExport {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017)
|
||||
static final Map retMap = new ConcurrentHashMap()
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
|
||||
// 在此更改时间
|
||||
static final def TRADE_DATE_START = "2020-03-28"
|
||||
|
||||
static main(args) {
|
||||
def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州",
|
||||
"合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台",
|
||||
"徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁",
|
||||
"南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华",
|
||||
"昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城",
|
||||
"芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州",
|
||||
"连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口",
|
||||
"西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"]
|
||||
// def cities = ["常州"]
|
||||
|
||||
def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"]
|
||||
List<String> head = new ArrayList<>()
|
||||
for (String name: headArr) {
|
||||
List<String> temp = new ArrayList<>()
|
||||
temp.add(name)
|
||||
head.add(temp)
|
||||
}
|
||||
|
||||
CountDownLatch latch = new CountDownLatch(cities.size())
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("city", 1)
|
||||
filter.put("region", 1)
|
||||
filter.put("district_name", 1)
|
||||
filter.put("total_price", 1)
|
||||
filter.put("trade_date", 1)
|
||||
filter.put("avg_price", 1)
|
||||
filter.put("area", 1)
|
||||
filter.put("direction", 1)
|
||||
filter.put("fitment", 1)
|
||||
filter.put("height", 1)
|
||||
filter.put("floor", 1)
|
||||
filter.put("unit_num", 1)
|
||||
filter.put("house_num", 1)
|
||||
filter.put("room_num", 1)
|
||||
filter.put("hall", 1)
|
||||
filter.put("room", 1)
|
||||
filter.put("toilet", 1)
|
||||
for (def city : cities) {
|
||||
final currCity = city
|
||||
executorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
final def ret = new CopyOnWriteArrayList()
|
||||
// 小区
|
||||
def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, obj)
|
||||
if (obj.alias) {
|
||||
// 小区可能存在多个别名
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"]
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put("city", currCity)
|
||||
query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))
|
||||
|
||||
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).count())
|
||||
fangjiaConn.find(query, filter).batchSize(10000).each { tempObj ->
|
||||
final obj = tempObj
|
||||
districtExecutorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
def row = []
|
||||
row[0] = currCity
|
||||
row[1] = obj.region
|
||||
row[2] = obj.district_name
|
||||
row[3] = obj.total_price
|
||||
row[4] = obj.avg_price
|
||||
if (obj.trade_date) {
|
||||
row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd")
|
||||
}
|
||||
row[6] = obj.area
|
||||
row[7] = obj.direction
|
||||
row[8] = obj.fitment
|
||||
row[9] = obj.height
|
||||
row[10] = obj.floor
|
||||
row[11] = obj.unit_num
|
||||
row[12] = obj.house_num
|
||||
row[13] = obj.room_num
|
||||
row[14] = obj.hall
|
||||
row[15] = obj.room
|
||||
row[16] = obj.toilet
|
||||
def objOur = map.get(obj.region + obj.district_name)
|
||||
calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row)
|
||||
ret.add(row)
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
println("出现异常111111.。。。。。。。。。。。")
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
districtLatch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
districtLatch.await()
|
||||
EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret);
|
||||
//ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息")
|
||||
println("当前处理城市" + currCity + " " + ret.size())
|
||||
} catch (Exception ex) {
|
||||
println("出现异常22222.。。。。。。。。。。。")
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
executorService.shutdown()
|
||||
districtExecutorService.shutdown()
|
||||
}
|
||||
|
||||
private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) {
|
||||
// 最相近一周的该小区均价
|
||||
Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7)
|
||||
Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7)
|
||||
|
||||
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
|
||||
dbObject.put("region", objOur.region)
|
||||
dbObject.put("district_name", objOur.name)
|
||||
// 不可用append,会被覆盖
|
||||
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
|
||||
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
Double weekendAvgPrice = 0
|
||||
dbCursor.each { o ->
|
||||
weekendAvgPrice = o.avg_price
|
||||
}
|
||||
|
||||
if (weekendAvgPrice == 0) {
|
||||
println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name)
|
||||
row[17] = ""
|
||||
row[18] = ""
|
||||
} else {
|
||||
row[17] = weekendAvgPrice
|
||||
row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice))
|
||||
}
|
||||
}
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang3.time.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 模型级联项目涉及城市
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/23
|
||||
*/
|
||||
class PPE15Export {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
|
||||
// 在此更改时间
|
||||
static final def TRADE_DATE_START = "2020-03-28"
|
||||
|
||||
static main(args) {
|
||||
def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
|
||||
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
|
||||
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
|
||||
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
|
||||
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
|
||||
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
|
||||
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
|
||||
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
//def cities = ["扬州"]
|
||||
def result = []
|
||||
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
|
||||
|
||||
CountDownLatch latch = new CountDownLatch(cities.size())
|
||||
for (def cityTemp : cities) {
|
||||
final city = cityTemp
|
||||
println("当前处理城市:" + city)
|
||||
executorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
AtomicInteger totalCount = new AtomicInteger(0)
|
||||
AtomicInteger count = new AtomicInteger(0)
|
||||
def row = []
|
||||
row[0] = city
|
||||
// 小区
|
||||
def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, obj)
|
||||
if (obj.alias) {
|
||||
// 小区可能存在多个别名
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
|
||||
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).count())
|
||||
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).each { obj ->
|
||||
districtExecutorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
def objOur = map.get(obj.region + obj.district_name)
|
||||
Double avgPrice = obj.avg_price
|
||||
Date tradeDate = obj.trade_date
|
||||
// 最相近一周的该小区均价
|
||||
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
|
||||
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
|
||||
|
||||
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
|
||||
dbObject.put("region", obj.region)
|
||||
dbObject.put("district_name", objOur.name)
|
||||
// 不可用append,会被覆盖
|
||||
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
|
||||
dbObject.put("region", obj.region)
|
||||
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
|
||||
// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name)
|
||||
// .append("weekend", new BasicDBObject('$lte', nextWeekDate))
|
||||
// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
Double weekendAvgPrice = 0
|
||||
dbCursor.each { o ->
|
||||
weekendAvgPrice = o.avg_price
|
||||
}
|
||||
|
||||
if (weekendAvgPrice == 0) {
|
||||
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
|
||||
} else {
|
||||
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
|
||||
if (ppe15) {
|
||||
count.incrementAndGet()
|
||||
}
|
||||
}
|
||||
totalCount.incrementAndGet()
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
districtLatch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
districtLatch.await()
|
||||
row[6] = totalCount.get()
|
||||
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
|
||||
retMap.put(city, row)
|
||||
|
||||
println("当前处理城市" + city + " " + row)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
latch.await()
|
||||
for (def city: cities) {
|
||||
if (retMap.get(city)) {
|
||||
result << retMap.get(city)
|
||||
} else {
|
||||
def row = []
|
||||
row[0] = city
|
||||
row[6] = 0
|
||||
row[7] = 0
|
||||
result << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
|
||||
executorService.shutdown()
|
||||
districtExecutorService.shutdown()
|
||||
}
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang3.time.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 模型级联项目涉及城市
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/23
|
||||
*/
|
||||
class PPE15ExportTest {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
|
||||
static main(args) {
|
||||
// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
|
||||
// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
|
||||
// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
|
||||
// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
|
||||
// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
|
||||
// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
|
||||
// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
|
||||
// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def cities = ["常州"]
|
||||
def result = []
|
||||
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
|
||||
|
||||
for (def cityTemp : cities) {
|
||||
final city = cityTemp
|
||||
println("当前处理城市:" + city)
|
||||
try {
|
||||
AtomicInteger totalCount = new AtomicInteger(0)
|
||||
AtomicInteger count = new AtomicInteger(0)
|
||||
def row = []
|
||||
row[0] = city
|
||||
// 小区
|
||||
final def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, obj)
|
||||
if (obj.alias) {
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
|
||||
int ppe15Count = 0
|
||||
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
|
||||
.batchSize(10000).count())
|
||||
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
|
||||
.batchSize(10000).each { obj ->
|
||||
try {
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
def objOur = map.get(obj.region + obj.district_name)
|
||||
Double avgPrice = obj.avg_price
|
||||
Date tradeDate = obj.trade_date
|
||||
// 最相近一周的该小区均价
|
||||
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
|
||||
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
|
||||
|
||||
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
|
||||
dbObject.put("region", obj.region)
|
||||
dbObject.put("district_name", objOur.name)
|
||||
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
|
||||
dbObject.put("region", obj.region)
|
||||
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
|
||||
Double weekendAvgPrice = 0
|
||||
dbCursor.each { o ->
|
||||
weekendAvgPrice = o.avg_price
|
||||
}
|
||||
|
||||
if (weekendAvgPrice == 0) {
|
||||
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
|
||||
} else {
|
||||
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
|
||||
if (ppe15) {
|
||||
count.incrementAndGet()
|
||||
}
|
||||
}
|
||||
totalCount.incrementAndGet()
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
districtLatch.countDown()
|
||||
}
|
||||
}
|
||||
districtLatch.await()
|
||||
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
|
||||
retMap.put(city, row)
|
||||
println("当前处理城市" + city + " " + row)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
|
||||
}
|
||||
};
|
||||
for (def city: cities) {
|
||||
if (retMap.get(city)) {
|
||||
result << retMap.get(city)
|
||||
} else {
|
||||
result << 0
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
|
||||
}
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 上海房屋租金导出
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/03/22
|
||||
*/
|
||||
class RentExport {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
|
||||
static main(args) {
|
||||
def result = []
|
||||
result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"]
|
||||
|
||||
int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
|
||||
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
|
||||
.batchSize(10000).count()
|
||||
AtomicInteger size = new AtomicInteger(count)
|
||||
println("上海小区" + size.get())
|
||||
CountDownLatch latch = new CountDownLatch(count)
|
||||
DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
|
||||
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.append("city", "上海").append("district_name", obj.name).append("region", obj.region)
|
||||
.append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22")))
|
||||
.append("house_type", new BasicDBObject('$lt', 5))
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1)
|
||||
.append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1)
|
||||
|
||||
buildResult(query, filter, result, obj.lng, obj.lat)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
println("*********************" + size.decrementAndGet())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
latch.await()
|
||||
result.addAll(retMap.values())
|
||||
com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金");
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
static boolean buildResult (def query, def filter, def result, def lng, def lat) {
|
||||
collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj ->
|
||||
def row = []
|
||||
row[0] = obj.city
|
||||
row[1] = obj.region
|
||||
row[2] = obj.district_name
|
||||
row[3] = obj.house_type
|
||||
row[4] = obj.avg_price
|
||||
row[5] = obj.sample_enum
|
||||
row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d)
|
||||
row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d)
|
||||
|
||||
if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) {
|
||||
// 判断价格是否合理
|
||||
if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) {
|
||||
retMap.put(obj.district_name + obj.region + obj.house_type, row)
|
||||
println(retMap.size())
|
||||
} else {
|
||||
println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) {
|
||||
if (retMap.contains(district_name + region + "1")) {
|
||||
printf("一居室" + retMap.get(district_name + region + "1")[4])
|
||||
}
|
||||
if (retMap.contains(district_name + region + "2")) {
|
||||
printf("二居室" + retMap.get(district_name + region + "2")[4])
|
||||
}
|
||||
if (retMap.contains(district_name + region + "3")) {
|
||||
printf("三居室" + retMap.get(district_name + region + "3")[4])
|
||||
}
|
||||
println("当前居室" + house_type + ", 价格为" + avg_price)
|
||||
switch (house_type) {
|
||||
case 2:
|
||||
println(retMap.get(district_name + region + "1"))
|
||||
if (retMap.get(district_name + region + "1")) {
|
||||
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
|
||||
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
if (retMap.get(district_name + region + "1")) {
|
||||
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
|
||||
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (retMap.get(district_name + region + "2")) {
|
||||
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
|
||||
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 4:
|
||||
if (retMap.get(district_name + region + "1")) {
|
||||
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
|
||||
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (retMap.get(district_name + region + "2")) {
|
||||
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
|
||||
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (retMap.get(district_name + region + "3")) {
|
||||
double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4])
|
||||
if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
+57
-1
@@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang3.time.DateFormatUtils
|
||||
|
||||
/**
|
||||
* ץȡ×ÜÁ¿¼à¿Ø
|
||||
@@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch {
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SeawaterMonitor(), args)
|
||||
//runBatch(new SeawaterMonitor(), args)
|
||||
//monitorDayCrawlReasonable()
|
||||
def results = []
|
||||
results << ["日期", "网站", "出售", "出租"]
|
||||
for (int i = 1; i < 90; i++) {
|
||||
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime()
|
||||
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
|
||||
countDayCrawlBySite(startDate, endDate, results)
|
||||
}
|
||||
ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1")
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch {
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计近90天更新量超过40万的天数
|
||||
* @param args
|
||||
*/
|
||||
static def monitorDayCrawlReasonable() {
|
||||
def map = [:]
|
||||
for (int i = 1; i < 90; i++) {
|
||||
def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i)
|
||||
def startDate = date.getTime();
|
||||
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
|
||||
def total = countDayCrawl(startDate, endDate)
|
||||
if (total >= expectNumber) {
|
||||
map.put(DateUtil.formatToYYYYMMDD(date), total)
|
||||
}
|
||||
}
|
||||
println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" )
|
||||
for (def key : map.keySet()) {
|
||||
println(key + " : " + map.get(key))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计每个网站时间段范围内的更新量
|
||||
*/
|
||||
static def countDayCrawlBySite(def startDate, def endDate, def results) {
|
||||
def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"]
|
||||
def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat"
|
||||
def map = [:]
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
map.put(data.root + "|" + data.cat, data.total)
|
||||
});
|
||||
for (def site : sites) {
|
||||
results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")]
|
||||
}
|
||||
return results
|
||||
};
|
||||
|
||||
static def countDayCrawl(def startDate, def endDate) {
|
||||
def total = 0
|
||||
def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'"
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
return total
|
||||
};
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user