Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+16
-4
@@ -106,7 +106,13 @@ public class EvaluationAuditController {
|
||||
if(eval!=null){
|
||||
bo.setHouseArea(eval.getHouseArea());
|
||||
bo.setDistrictName(eval.getDistrictName());
|
||||
bo.setAddress(eval.getAddress());
|
||||
String address = "";
|
||||
if(StringUtils.isNotBlank(eval.getCertificateAddress())){
|
||||
address = eval.getCertificateAddress();
|
||||
}else{
|
||||
address = eval.getAddress();
|
||||
}
|
||||
bo.setAddress(address);
|
||||
bo.setHouseNum(eval.getHouseNum());
|
||||
bo.setRoomNum(eval.getRoomNum());
|
||||
bo.setEvaluationDate(eval.getLastUpdated());
|
||||
@@ -115,7 +121,7 @@ public class EvaluationAuditController {
|
||||
bo.setFloorDesc(eval.getFloorDesc());
|
||||
bos.add(bo);
|
||||
|
||||
List<FjtEvaluation> evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), eval.getAddress(), eval.getHouseNum(), eval.getRoomNum());
|
||||
List<FjtEvaluation> evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), bo.getAddress(), eval.getHouseNum(), eval.getRoomNum());
|
||||
if(evaluations!=null && evaluations.size()>=2){
|
||||
List<String> ids = new ArrayList<String>();
|
||||
for(FjtEvaluation fjtEval : evaluations){
|
||||
@@ -148,7 +154,13 @@ public class EvaluationAuditController {
|
||||
BeanUtils.copyProperties(audit, bo);
|
||||
bo.setHouseArea(eval.getHouseArea());
|
||||
bo.setDistrictName(eval.getDistrictName());
|
||||
bo.setAddress(eval.getAddress());
|
||||
String address = "";
|
||||
if(StringUtils.isNotBlank(eval.getCertificateAddress())){
|
||||
address = eval.getCertificateAddress();
|
||||
}else{
|
||||
address = eval.getAddress();
|
||||
}
|
||||
bo.setAddress(address);
|
||||
bo.setHouseNum(eval.getHouseNum());
|
||||
bo.setRoomNum(eval.getRoomNum());
|
||||
bo.setEvaluationDate(eval.getLastUpdated());
|
||||
@@ -157,7 +169,7 @@ public class EvaluationAuditController {
|
||||
bo.setFloorDesc(eval.getFloorDesc());
|
||||
bos.add(bo);
|
||||
//根据地址,楼栋号判断是否是同一套房
|
||||
List<FjtEvaluation> evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), eval.getAddress(), eval.getHouseNum(), eval.getRoomNum());
|
||||
List<FjtEvaluation> evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), bo.getAddress(), eval.getHouseNum(), eval.getRoomNum());
|
||||
if(evaluations!=null && evaluations.size()>=2){
|
||||
List<String> ids = new ArrayList<String>();
|
||||
for(FjtEvaluation fjtEval : evaluations){
|
||||
|
||||
+2
-2
@@ -362,8 +362,8 @@ public class YugudanController {
|
||||
if (model.getId()>0) {
|
||||
//FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(model.getId());
|
||||
Map<String, Object> data = new HashMap<String, Object>();
|
||||
if(StringUtils.isNotBlank(model.getAddress())){
|
||||
data.put("address", model.getAddress());
|
||||
if(StringUtils.isNotBlank(model.getCertificateAddress())){
|
||||
data.put("certificateAddress", model.getCertificateAddress());
|
||||
}
|
||||
// if(model.getPeopleEditDate()!=null || model.getPeopleEditPrice()!=null){
|
||||
// data.put("peopleEdit", 1);
|
||||
|
||||
+3
-1
@@ -113,7 +113,9 @@ public class fjtEvaluationController {
|
||||
FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Integer.parseInt(bo.getEvaluationId()));
|
||||
if(evaluation != null){
|
||||
String addr = "";
|
||||
if(evaluation.getAddress() != null){
|
||||
if(StringUtils.isNotBlank(evaluation.getCertificateAddress())){
|
||||
addr += evaluation.getCertificateAddress();
|
||||
}else if(evaluation.getAddress() != null){
|
||||
addr += evaluation.getAddress();
|
||||
}
|
||||
if(StringUtils.isNotBlank(evaluation.getHouseNum())){
|
||||
|
||||
@@ -150,8 +150,10 @@ public class YugudanModel extends BaseModel{
|
||||
private String reportNum;
|
||||
|
||||
private Integer askPriceId;//人工询价id
|
||||
|
||||
|
||||
|
||||
private String certificateAddress;
|
||||
|
||||
|
||||
public int getId() {
|
||||
return id;
|
||||
}
|
||||
@@ -861,7 +863,12 @@ public class YugudanModel extends BaseModel{
|
||||
public void setUnitNum(String unitNum) {
|
||||
this.unitNum = unitNum;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public String getCertificateAddress() {
|
||||
return certificateAddress;
|
||||
}
|
||||
|
||||
public void setCertificateAddress(String certificateAddress) {
|
||||
this.certificateAddress = certificateAddress;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,19 +724,59 @@
|
||||
});
|
||||
var series = new Array();
|
||||
series[0] = {
|
||||
name:"房源散列图",
|
||||
name:"房源散列图-普通住宅",
|
||||
data:new Array()
|
||||
//color:"#660099"
|
||||
}
|
||||
series[1] = {
|
||||
name:"房源散列图-商住",
|
||||
data:new Array(),
|
||||
color:"#12990f"
|
||||
}
|
||||
series[2] = {
|
||||
name:"房源散列图-别墅",
|
||||
data:new Array(),
|
||||
color:"#992632"
|
||||
}
|
||||
var series_arry_0 = []
|
||||
var series_arry_1 = []
|
||||
var series_arry_2 = []
|
||||
$(data).each(function(i,item){
|
||||
var pdate = item.pDate + "";
|
||||
var ds = pdate.substring(0, 4) + '-' + pdate.substring(4, 6) + '-' + pdate.substring(6)
|
||||
series[0].data.push({
|
||||
x:new Date(ds),
|
||||
y:Math.round(item.totalPrice/item.area),
|
||||
data:item
|
||||
})
|
||||
if ((item.estateType!=null) && (item.estateType.indexOf('别墅') > -1)){
|
||||
series_arry_2.push({
|
||||
x:new Date(ds),
|
||||
y:Math.round(item.totalPrice/item.area),
|
||||
data:item
|
||||
})
|
||||
}else if((item.estateType!=null) && ((item.estateType.indexOf('商住') > -1) || (item.estateType.indexOf('公寓酒店') > -1))){
|
||||
series_arry_1.push({
|
||||
x:new Date(ds),
|
||||
y:Math.round(item.totalPrice/item.area),
|
||||
data:item
|
||||
})
|
||||
}else{
|
||||
series_arry_0.push({
|
||||
x:new Date(ds),
|
||||
y:Math.round(item.totalPrice/item.area),
|
||||
data:item
|
||||
})
|
||||
}
|
||||
// series[0].data.push({
|
||||
// x:new Date(ds),
|
||||
// y:Math.round(item.totalPrice/item.area),
|
||||
// data:item
|
||||
// })
|
||||
// series[1].data.push({
|
||||
// x:new Date(ds),
|
||||
// y:Math.round(item.totalPrice/item.area),
|
||||
// data:item
|
||||
// })
|
||||
});
|
||||
series[0].data = series_arry_0
|
||||
series[1].data = series_arry_1
|
||||
series[2].data = series_arry_2
|
||||
$('#houseChart').highcharts({
|
||||
chart: {
|
||||
type: 'scatter',
|
||||
|
||||
@@ -152,7 +152,7 @@
|
||||
<tbody>
|
||||
<#list loudongList! as loudong>
|
||||
<tr>
|
||||
<td width="4%">
|
||||
<td width="9%">
|
||||
${loudong.districtNo!''}
|
||||
</td>
|
||||
<td width="4%">
|
||||
|
||||
@@ -140,6 +140,7 @@
|
||||
<th style="width:60px;">板块</th>
|
||||
<th style="width:80px;">装修</th>
|
||||
<th style="width:70px;">朝向</th>
|
||||
<th style="width:70px;">物业类型</th>
|
||||
<th style="width:90px;">来源</th>
|
||||
<th style="width:80px;">操作</th>
|
||||
</tr>
|
||||
@@ -180,6 +181,7 @@
|
||||
<td>${it.block!}</td>
|
||||
<td>${it.fitment!}</td>
|
||||
<td>${it.direction!}</td>
|
||||
<td>${it.estateType!}</td>
|
||||
<td>
|
||||
<#if it.url??>
|
||||
<a href="${it.url!}" target="_blank" title="${it.url!}">${t_util.urlToSiteName(it.url!)!'未知'}</a>
|
||||
|
||||
@@ -241,10 +241,10 @@
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class = "t1">物业地址</td>
|
||||
<td class = "t1">产证地址</td>
|
||||
<td>
|
||||
<span class="labelModel1">${(evaluation.address)!}</span>
|
||||
<span class="editModel1 hide"><input type="text" name="address" value="${(evaluation.address)!}"/></span>
|
||||
<span class="labelModel1"><#if evaluation?? && evaluation.certificateAddress??>${(evaluation.certificateAddress)!}<#else>${(evaluation.address)!}</#if></span>
|
||||
<span class="editModel1 hide"><input type="text" name="certificateAddress" value="<#if evaluation?? && evaluation.certificateAddress??>${(evaluation.certificateAddress)!}<#else>${(evaluation.address)!}</#if>"/></span>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
<th width="6%">城市</th>
|
||||
<th width="6%">区域</th>
|
||||
<th width="10%">楼盘名称</th>
|
||||
<th width="14%">估价地址</th>
|
||||
<th width="14%">产证地址</th>
|
||||
<th width="10%">帐号</th>
|
||||
<th width="6%">面积</th>
|
||||
<th width="10%">总价</th>
|
||||
@@ -83,7 +83,7 @@
|
||||
<td>${it.city}</td>
|
||||
<td>${it.region!}</td>
|
||||
<td><a href="../yugudan/editEval?id=${it.id}">${it.districtName!}</a></td>
|
||||
<td>${it.address!}<#if it.houseNum??>${it.houseNum!}号</#if>
|
||||
<td><#if it.certificateAddress??>${(it.certificateAddress)!}<#else>${(it.address)!}</#if><#if it.houseNum??>${it.houseNum!}号</#if>
|
||||
<#if it.unitNum?? && it.unitNum?length gt 1>${it.unitNum!}单元</#if>
|
||||
<#if it.buildType!=3 && it.buildType!=8 && it.buildType!=0>
|
||||
<#if it.floorDesc?? && it.floorDesc?length gt 1>${it.floorDesc!}层</#if>
|
||||
|
||||
@@ -1341,6 +1341,7 @@ public class DataUtil {
|
||||
GZIPOutputStream outputStream = new GZIPOutputStream(bao);
|
||||
outputStream.write(content.getBytes("UTF-8"));
|
||||
outputStream.finish();
|
||||
System.out.println(bao.toString());
|
||||
return bao.toByteArray();
|
||||
}
|
||||
|
||||
|
||||
+87
-2
@@ -6,12 +6,20 @@ import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Handler;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.DingDingMessageUtil;
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.core.util.HttpRequestException;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fangjia.coresite.model.JsonModel;
|
||||
@@ -27,9 +35,12 @@ import com.fangjia.service.common.CookieName;
|
||||
import com.fangjia.service.fangjia.NewsResourceService;
|
||||
import com.fangjia.service.fangjia.SaleLeadsService;
|
||||
import com.fangjia.service.mongo.fangjia.ServerAreaService;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
public class SaleLeadsController {
|
||||
|
||||
private static final Logger LOG = Logger.getLogger(SaleLeadsController.class);
|
||||
|
||||
private SaleLeadsService saleLeadsService;
|
||||
private ServerAreaService serverAreaService;
|
||||
private NewsResourceService newsResourceService;
|
||||
@@ -41,6 +52,15 @@ public class SaleLeadsController {
|
||||
return model;
|
||||
}
|
||||
|
||||
/**
|
||||
* 地数公众号咨询
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object didataApplyAction(SaleLeadsModel model) {
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object doAddAction(SaleLeadsModel model, HttpServletRequest req,
|
||||
HttpServletResponse res) throws UnsupportedEncodingException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
|
||||
|
||||
@@ -129,8 +149,64 @@ public class SaleLeadsController {
|
||||
}
|
||||
|
||||
saleLeads = converValue(saleLeads);
|
||||
String body =
|
||||
"业务需求:" + saleLeads.getBusiness() + " " + saleLeads.getOtherBusiness() + "\n\n"
|
||||
+ "姓名:" + saleLeads.getUname() + "\n\n"
|
||||
+ "城市:" + saleLeads.getCity() + "\n\n"
|
||||
+ "工作职能:" + saleLeads.getWork() + " " + saleLeads.getOtherWork() + "\n\n"
|
||||
+ "手机:" + saleLeads.getMobile() + "\n\n"
|
||||
+ "邮箱:" + saleLeads.getEmail() + "\n\n"
|
||||
+ "公司名称:" + saleLeads.getCompany() + "\n\n"
|
||||
+ "行业性质:" + saleLeads.getIndustryNature() + " " + saleLeads.getOtherIndustryNature() + "\n\n";
|
||||
body = body.replaceAll("null", "").replaceAll("其他,请填写", "");
|
||||
if (StringUtils.isNotBlank(refUrl) && refUrl.contains("bd_vid")) {
|
||||
body += "线索来源:百度";
|
||||
} else if (StringUtils.isNotBlank(refUrl) && refUrl.contains("didata.wechat")) {
|
||||
body += "线索来源: 地数公众号";
|
||||
} else {
|
||||
body += "线索来源:房价网";
|
||||
}
|
||||
try {
|
||||
JSONObject requestBody = new JSONObject();
|
||||
requestBody.put("msgtype", "text");
|
||||
JSONObject markdown = new JSONObject();
|
||||
markdown.put("title", "销售线索");
|
||||
markdown.put("content", body);
|
||||
requestBody.put("text", markdown);
|
||||
HttpRequest request = new HttpRequest();
|
||||
final String ding_url = "https://oapi.dingtalk.com/robot/send?access_token=894650a12af9497bed599df1d5f93b21d6da5bbae45f35f18c5825580ebbdf64";
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Content-Type", "application/json; charset=utf-8");
|
||||
request.postWithBody(ding_url, requestBody, headers, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
saleLeadsService.save(saleLeads);
|
||||
|
||||
|
||||
// 对于百度广告过来的信息,进行回传
|
||||
try {
|
||||
if (StringUtils.isNotBlank(refUrl) && refUrl.contains("bd_vid")) {
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
final String baiduUrl = "https://ocpc.baidu.com/ocpcapi/api/uploadConvertData";
|
||||
Map<String, String> bdHeaders = new HashMap<String, String>();
|
||||
bdHeaders.put("Content-Type", "application/json; charset=utf-8");
|
||||
|
||||
JSONObject requestBody = new JSONObject();
|
||||
// https://fengchao.baidu.com/fc/toolscenter/optimize/track/user/27002539
|
||||
requestBody.put("token", "EZGISwlOClRMEZErT2ThZTEjpLAzs2wE@9POwrGLqpRtQeGhvvCug1N3OK1tq3skk");
|
||||
JSONArray jsonArray = new JSONArray();
|
||||
JSONObject contentJson = new JSONObject();
|
||||
contentJson.put("logidUrl", refUrl);
|
||||
contentJson.put("newType", "3");
|
||||
jsonArray.add(contentJson);
|
||||
requestBody.put("conversionTypes", jsonArray);
|
||||
String result = httpRequest.postWithBody(baiduUrl, requestBody, bdHeaders, "UTF-8");
|
||||
LOG.info("百度回传成功:" + result);
|
||||
}
|
||||
} catch (HttpRequestException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
if(StringUtils.isNotBlank(saleLeads.getBusiness()) && saleLeads.getBusiness().contains("_")){
|
||||
String[] businessArray = saleLeads.getBusiness().split("_");
|
||||
for(String s : businessArray){
|
||||
@@ -155,7 +231,12 @@ public class SaleLeadsController {
|
||||
|
||||
if(StringUtils.isNotBlank(refUrl) && refUrl.contains("/resource/info")){
|
||||
url = refUrl.substring(refUrl.indexOf("/resource/info"), refUrl.length());
|
||||
}else{
|
||||
} else if (StringUtils.isNotBlank(refUrl) && refUrl.contains("didata.wechat")) { // 地数公众号跳转
|
||||
url = "/enquire/didataSuccess";
|
||||
if (StringUtils.isNotBlank(refUrl)) {
|
||||
url = url + "?backUrl=" + URLEncoder.encode(refUrl);
|
||||
}
|
||||
} else{
|
||||
url = "/enquire/success";
|
||||
if (StringUtils.isNotBlank(refUrl)) {
|
||||
url = url + "?backUrl=" + URLEncoder.encode(refUrl);
|
||||
@@ -224,6 +305,10 @@ public class SaleLeadsController {
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object didataResultAction(SaleLeadsModel model) {
|
||||
return model;
|
||||
}
|
||||
|
||||
public SaleLeadsService getSaleLeadsService() {
|
||||
return saleLeadsService;
|
||||
}
|
||||
|
||||
+10
-1
@@ -1,6 +1,15 @@
|
||||
package com.fangjia.coresite.controllers;
|
||||
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.core.util.HttpRequestException;
|
||||
import com.fangjia.coresite.model.FangjiaModel;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 解决方案 控制器
|
||||
@@ -14,7 +23,7 @@ public class SolutionController {
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object apiAction(FangjiaModel model) {
|
||||
public Object apiAction(FangjiaModel model, HttpServletRequest request) {
|
||||
return model;
|
||||
}
|
||||
|
||||
|
||||
+5
@@ -48,6 +48,11 @@ public class AntiSpiderFilter implements Filter {
|
||||
|
||||
HttpServletRequest request = (HttpServletRequest) req;
|
||||
HttpServletResponse response = (HttpServletResponse) resp;
|
||||
// 放开首页不做爬虫验证
|
||||
if (request.getRequestURL().toString().endsWith("fangjia.com") || request.getRequestURL().toString().endsWith("fangjia.com/")) {
|
||||
chain.doFilter(req, resp);
|
||||
return;
|
||||
}
|
||||
if(request.getRequestURI().endsWith("/validate") || request.getRequestURI().contains("randomCode") || request.getRequestURI().contains("checkCode")|| request.getRequestURI().contains("yeepayCallBack")){
|
||||
chain.doFilter(req, resp);
|
||||
return;
|
||||
|
||||
@@ -309,7 +309,17 @@
|
||||
<ladder>/</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
|
||||
|
||||
<rule controller="saleLeads" action="didataApply" channel="enquire">
|
||||
<ladders>
|
||||
<ladder>/didataApply</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
<rule controller="saleLeads" action="didataResult" channel="enquire">
|
||||
<ladders>
|
||||
<ladder>/didataSuccess</ladder>
|
||||
</ladders>
|
||||
</rule>
|
||||
|
||||
<rule controller="saleLeads" action="result" channel="enquire">
|
||||
<ladders>
|
||||
|
||||
@@ -153,46 +153,47 @@
|
||||
</div>
|
||||
<div class="main_title" id="partner" style="padding-top:100px;">他们都在和我们合作</div>
|
||||
<div class="main_content partner mobile01">
|
||||
<a class="partner11"></a>
|
||||
<a class="partner12"></a>
|
||||
<a class="partner13"></a>
|
||||
<#-- <a class="partner14"></a>-->
|
||||
<a class="partner15"></a>
|
||||
<a class="partner16"></a>
|
||||
<a class="partner21"></a>
|
||||
<a class="partner22"></a>
|
||||
<a class="partner23"></a>
|
||||
<a class="partner24"></a>
|
||||
<a class="partner25"></a>
|
||||
<a class="partner26"></a>
|
||||
<a class="partner31"></a>
|
||||
<a class="partner32"></a>
|
||||
<a class="partner33"></a>
|
||||
<a class="partner34"></a>
|
||||
<a class="partner35"></a>
|
||||
<a class="partner36"></a>
|
||||
<a class="partner41"></a>
|
||||
<a class="partner42"></a>
|
||||
<a class="partner43"></a>
|
||||
<a class="partner44"></a>
|
||||
<a class="partner45"></a>
|
||||
<a class="partner46"></a>
|
||||
<a class="partner51"></a>
|
||||
<a class="partner52"></a>
|
||||
<a class="partner53"></a>
|
||||
<a class="partner54"></a>
|
||||
<a class="partner55"></a>
|
||||
<a class="partner56"></a>
|
||||
<a class="partner61"></a>
|
||||
<a class="partner62"></a>
|
||||
<a class="partner63"></a>
|
||||
<a class="partner65"></a>
|
||||
<a class="partner66"></a>
|
||||
<a class="partner72"></a>
|
||||
<a class="partner73"></a>
|
||||
<img src="${staticUrl}/images/fangjia/v5/company.png" style="width: 99%;"><br>
|
||||
<#-- <a class="partner11"></a>-->
|
||||
<#-- <a class="partner12"></a>-->
|
||||
<#-- <a class="partner13"></a>-->
|
||||
<#--<#– <a class="partner14"></a>–>-->
|
||||
<#-- <a class="partner15"></a>-->
|
||||
<#-- <a class="partner16"></a>-->
|
||||
<#-- <a class="partner21"></a>-->
|
||||
<#-- <a class="partner22"></a>-->
|
||||
<#-- <a class="partner23"></a>-->
|
||||
<#-- <a class="partner24"></a>-->
|
||||
<#-- <a class="partner25"></a>-->
|
||||
<#-- <a class="partner26"></a>-->
|
||||
<#-- <a class="partner31"></a>-->
|
||||
<#-- <a class="partner32"></a>-->
|
||||
<#-- <a class="partner33"></a>-->
|
||||
<#-- <a class="partner34"></a>-->
|
||||
<#-- <a class="partner35"></a>-->
|
||||
<#-- <a class="partner36"></a>-->
|
||||
<#-- <a class="partner41"></a>-->
|
||||
<#-- <a class="partner42"></a>-->
|
||||
<#-- <a class="partner43"></a>-->
|
||||
<#-- <a class="partner44"></a>-->
|
||||
<#-- <a class="partner45"></a>-->
|
||||
<#-- <a class="partner46"></a>-->
|
||||
<#-- <a class="partner51"></a>-->
|
||||
<#-- <a class="partner52"></a>-->
|
||||
<#-- <a class="partner53"></a>-->
|
||||
<#-- <a class="partner54"></a>-->
|
||||
<#-- <a class="partner55"></a>-->
|
||||
<#-- <a class="partner56"></a>-->
|
||||
<#-- <a class="partner61"></a>-->
|
||||
<#-- <a class="partner62"></a>-->
|
||||
<#-- <a class="partner63"></a>-->
|
||||
<#-- <a class="partner65"></a>-->
|
||||
<#-- <a class="partner66"></a>-->
|
||||
<#-- <a class="partner72"></a>-->
|
||||
<#-- <a class="partner73"></a>-->
|
||||
</div>
|
||||
<div class="main_content partner mobile02">
|
||||
<img src="${staticUrl}/images/fangjia/v5/partner_m.png">
|
||||
<img src="${staticUrl}/images/fangjia/v5/partner_m1.png">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -207,29 +207,29 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="list">
|
||||
<div class="listTop">
|
||||
<p><img src="${imgUrl}/images/fangjia/v5/diamond.png"><label>合作伙伴</label></p>
|
||||
</div>
|
||||
<div class="contentBox">
|
||||
<a class="content" href="${basePath}/terms/ydinfo" style="border-bottom:none;">
|
||||
<img src="${imgUrl}/images/fangjia/v5/youda.jpeg">
|
||||
<div>
|
||||
<p><em>房价网合作伙伴-上海友达土地房地产评估有限公司简介</em></p>
|
||||
<p><span>上海友达土地房地产评估有限公司成立于1995年,是我国最早成立的独立从事房地产估价、土地估价、房地产咨询业务的专业机构之一。</span></p>
|
||||
<p><small>上海房产门户 </small></p>
|
||||
</div>
|
||||
</a>
|
||||
<a class="content" href="${basePath}/terms/ydintr">
|
||||
<img src="${imgUrl}/images/fangjia/v5/youda.jpeg">
|
||||
<div>
|
||||
<p><em>上海友达土地房地产评估有限公司-企业内部培训</em></p>
|
||||
<p><span>2021年8月9日,上海友达土地房地产评估有限公司就房地产征收业务中遇到的诸多问题开展了企业内部培训。本次培训的宗旨是为了进一步提高员工技术水平和工作效率,强化团队合作意识,为公司的可持续性发展奠定良好基础。</span></p>
|
||||
<p><small>上海房产门户 </small></p>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<#--<div class="list">-->
|
||||
<#-- <div class="listTop">-->
|
||||
<#-- <p><img src="${imgUrl}/images/fangjia/v5/diamond.png"><label>合作伙伴</label></p>-->
|
||||
<#-- </div>-->
|
||||
<#-- <div class="contentBox">-->
|
||||
<#-- <a class="content" href="${basePath}/terms/ydinfo" style="border-bottom:none;">-->
|
||||
<#-- <img src="${imgUrl}/images/fangjia/v5/youda.jpeg">-->
|
||||
<#-- <div>-->
|
||||
<#-- <p><em>房价网合作伙伴-上海友达土地房地产评估有限公司简介</em></p>-->
|
||||
<#-- <p><span>上海友达土地房地产评估有限公司成立于1995年,是我国最早成立的独立从事房地产估价、土地估价、房地产咨询业务的专业机构之一。</span></p>-->
|
||||
<#-- <p><small>上海房产门户 </small></p>-->
|
||||
<#-- </div>-->
|
||||
<#-- </a>-->
|
||||
<#-- <a class="content" href="${basePath}/terms/ydintr">-->
|
||||
<#-- <img src="${imgUrl}/images/fangjia/v5/youda.jpeg">-->
|
||||
<#-- <div>-->
|
||||
<#-- <p><em>上海友达土地房地产评估有限公司-企业内部培训</em></p>-->
|
||||
<#-- <p><span>2021年8月9日,上海友达土地房地产评估有限公司就房地产征收业务中遇到的诸多问题开展了企业内部培训。本次培训的宗旨是为了进一步提高员工技术水平和工作效率,强化团队合作意识,为公司的可持续性发展奠定良好基础。</span></p>-->
|
||||
<#-- <p><small>上海房产门户 </small></p>-->
|
||||
<#-- </div>-->
|
||||
<#-- </a>-->
|
||||
<#-- </div>-->
|
||||
<#--</div>-->
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -555,27 +555,53 @@ $(".user_btn").click(function(){
|
||||
scrollTo(0,0);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 解决信息丢失问题
|
||||
var work = $(".duty:visible").find("select").val()
|
||||
if (work) {
|
||||
work = work.replaceAll("请选择您的工作职能", "")
|
||||
work = work.replaceAll("请选择", "")
|
||||
$("#work").val(work);
|
||||
}
|
||||
|
||||
var box = $(".demand p i.checked");
|
||||
if(box.length>0) {
|
||||
$(".demand").parent().find(".error span").text("")
|
||||
$(".demand").parent().find(".error").hide();
|
||||
var selectOtherFlag = false;
|
||||
var content = []
|
||||
box.each(function () {
|
||||
var s = $(this).parent().find("span").text();
|
||||
if (s.indexOf("请填写") > 0) {
|
||||
selectOtherFlag = true;
|
||||
} else {
|
||||
content.push(s);
|
||||
}
|
||||
})
|
||||
$("#business").val(content.join("_"));
|
||||
}
|
||||
|
||||
|
||||
/*if(!checkEmail()){
|
||||
scrollBy(0,-300);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!checkNature()){
|
||||
scrollBy(0,-200);
|
||||
return false;
|
||||
}
|
||||
if(!checkWork()){
|
||||
scrollBy(0,-100);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
if(!checkNature()){
|
||||
scrollBy(0,-200);
|
||||
return false;
|
||||
}
|
||||
if(mid=="1018"||mid=="1016"){
|
||||
if(!checkOtherInfo()){
|
||||
scrollBy(0,-100);
|
||||
return false;
|
||||
}
|
||||
} */
|
||||
}
|
||||
*/
|
||||
|
||||
var cls = $(".agreement i").attr("class");
|
||||
if(cls.indexOf("checked")>-1){
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
|
||||
<title>时空价值挖掘平台 -地数 didata.ai</title>
|
||||
<meta name="keywords" content="地址数据智能处理,时空价值计算,时空数据统计分析,自动化客户分类建模">
|
||||
<meta name="description" content="提交联系信息,估家科技将为您提供详尽的产品功能及服务介绍">
|
||||
<#include "../jsconstants.ftl" />
|
||||
|
||||
<link href="${staticUrl}/style/fangjia/v5/common.css?v=${version}" rel="stylesheet">
|
||||
<link href="${staticUrl}/style/fangjia/v5/main.css?v=${version}" rel="stylesheet">
|
||||
<link href="${staticUrl}/style/fangjia/v5/city.css?v=${version}" rel="stylesheet">
|
||||
<link href="${staticUrl}/style/fangjia/v5/user.css?v=${version}" rel="stylesheet">
|
||||
<script src="${staticUrl}/js/fangjia/v5/jquery-1.7.2.min.js?v=${version}"></script>
|
||||
|
||||
<style>
|
||||
.inputInfoBox01 .inputInfo label.label01a{margin-top:10px;}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
if((navigator.userAgent.match(/(iPad|iPhone|iPod|Android|ios)/i))){
|
||||
document.write("<link href='${staticUrl}/style/fangjia/v5/sm.css?v=${version}' rel='stylesheet'>");
|
||||
document.write("<link href='${staticUrl}/style/fangjia/v5/main-m.css?v=${version}' rel='stylesheet'>");
|
||||
document.write("<link href='${staticUrl}/style/fangjia/v5/city-m.css?v=${version}' rel='stylesheet'>");
|
||||
document.write("<link href='${staticUrl}/style/fangjia/v5/theme-orange.css?v=${version}' rel='stylesheet'>");
|
||||
document.write("<link href='${staticUrl}/style/fangjia/v5/mobile.css?v=${version}' rel='stylesheet'>");
|
||||
}
|
||||
</script>
|
||||
|
||||
<style>
|
||||
@media(max-device-width:768px){
|
||||
.main{padding-top:0;}
|
||||
.main_left{display:none;}
|
||||
.main_right{float:none;width:100%;}
|
||||
.user{width:100%;margin-top:6rem;}
|
||||
.user02{margin-left:0;}
|
||||
.title{text-align:center;position:static;}
|
||||
.title p{margin:1rem;}
|
||||
.content{padding:0 1rem;padding-top:2rem;}
|
||||
.content03 .inputInfoBox{overflow:visible;}
|
||||
.inputInfoBox01{float:none;}
|
||||
.inputInfoBox01 .inputInfo{width:100%;height:40px;margin-bottom:20px;clear:both;}
|
||||
.inputInfoBox01 .inputInfo label{float:left;line-height:40px;}
|
||||
.inputInfoBox01 .inputInfo label.label01{float:left;line-height:20px;width:66px;}
|
||||
.inputInfoBox01 .inputInfo label.label01a{clear:both;margin-bottom:17px;margin-top:17px;}
|
||||
.inputInfoBox01 .inputInfo label i{margin-left:5px;}
|
||||
.inputInfoBox01a{margin-right:0;}
|
||||
.inpu,.sel{height:40px;line-height:40px;}
|
||||
.inpu,.sel,.demand{width:89.5%;float:right;}
|
||||
.dialogBox .inpu{float:left;}
|
||||
.inpu01{width:70%;}
|
||||
.inpu02a{margin-top:17px;}
|
||||
#selectedCity{width:auto;}
|
||||
.error01{top:-17px;left:78px;}
|
||||
.user_btn,.line{width:100%;}
|
||||
.footerM{display:none;}
|
||||
}
|
||||
@media(max-device-width:414px){
|
||||
.inpu,.sel,.demand{width:79.5%;}
|
||||
.inpu01{width:45%;}
|
||||
}
|
||||
@media(max-device-width:375px){
|
||||
.inpu,.sel,.demand{width:77%;}
|
||||
.inpu01{width:40%;}
|
||||
}
|
||||
@media(max-device-width:320px){
|
||||
.inpu,.sel,.demand{width:73%;}
|
||||
.inpu01{width:30%;}
|
||||
.demand p span{width:80%;}
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<#escape x as x?html>
|
||||
<div class="header">
|
||||
<div style="background-color: black;">
|
||||
<div class="logo" style="padding-top:2px;">
|
||||
<img class="img-box__img4" src="http://www.didata.ai/upload/20220522/3324a05faa1a88d483572ec64ac48c82.png"></img>
|
||||
<#-- <a href="http://www.didata.ai/upload/20220522/3324a05faa1a88d483572ec64ac48c82.png"></a>-->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="main">
|
||||
<div class="main_right">
|
||||
<div class="user user02">
|
||||
<div class="title">
|
||||
<p><label>预约产品Demo申请</label></p>
|
||||
<div style="text-align:left; margin-left: 2rem; font-size: 18px">我们很乐意为您提供帮助</div>
|
||||
<div style="text-align:left; margin-left: 3rem">√ 详尽的产品功能及服务介绍</div>
|
||||
<div style="text-align:left; margin-left: 3rem">√ 针对您所处的行业与业务场景,提供专业的服务方案<br/></div>
|
||||
<div style="text-align:left; margin-left: 3rem">√ 根据您的业务规模和需求,提供优惠的价格方案<br/></div>
|
||||
<div style="text-align:left; margin-left: 3rem">紧急咨询可拨打<strong>400-188-3787</strong><br/></div>
|
||||
<div style="text-align:left; margin-left: 3rem">预约申请请留下您的联系方式,我们会尽快与您联系<br/></div>
|
||||
</div>
|
||||
<div class="content content03">
|
||||
<form action="${wwwUrl}/saleLeads/doAdd" method="POST" id="_add_form">
|
||||
<input type="hidden" name="mid" value="${mid!}"/>
|
||||
<input type="hidden" name="downUrl" value="${downUrl!}"/>
|
||||
<input type="hidden" name="backUrl" value="${backUrl!}"/>
|
||||
<input type="hidden" name="resourceId" value="${resourceId!}"/>
|
||||
<div class="inputInfoBox">
|
||||
<div class="inputInfoBox01 inputInfoBox01a">
|
||||
<div class="inputInfo">
|
||||
<label>您的姓名<i>*</i></label>
|
||||
<input class="inpu inpu02" name="saleLeads.uname" id="uname" type="text" autocomplete="off" placeholder="请输入您的姓名">
|
||||
<div class="error error01 dn"><i></i><span></span></div>
|
||||
</div>
|
||||
<div class="inputInfo">
|
||||
<label>公司名称<i>*</i></label>
|
||||
<input class="inpu inpu02" name="saleLeads.company" id="company" type="text" autocomplete="off" placeholder="请输入您的公司名称">
|
||||
<div class="error error01 dn"><i></i><span></span></div>
|
||||
</div>
|
||||
<div class="inputInfo">
|
||||
<label><a class="ls2_01">手</a><a class="ls2_02">机</a><i>*</i></label>
|
||||
<input class="inpu inpu02" type="text" name="saleLeads.mobile" id="mobile" autocomplete="off" placeholder="请输入您的手机号码">
|
||||
<div class="error error01 dn"><i></i><span></span></div>
|
||||
</div>
|
||||
<div class="inputInfo">
|
||||
<label><a class="ls2_01">留</a><a class="ls2_02">言</a><i></i></label>
|
||||
<textarea class="inpu inpu02" scoll="auto" style="height:60px;padding-top:2px;" name="saleLeads.business" id="business">
|
||||
</textarea>
|
||||
<div class="error error01 dn"><i></i><span></span></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<a class="user_btn" style="background: #6db5ea">提交</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<#include "../citys.ftl"/>
|
||||
|
||||
<style>
|
||||
@media(max-device-width:768px){
|
||||
.main{padding-top:0;}
|
||||
.main_left{display:none;}
|
||||
.main_right{float:none;width:100%;}
|
||||
.user{width:100%;margin-top:6rem;}
|
||||
.user02{margin-left:0;}
|
||||
.title{text-align:center;position:static;}
|
||||
.title p{margin:1rem;}
|
||||
.content{padding:0 1rem;padding-top:2rem;}
|
||||
.content03 .inputInfoBox{overflow:visible;}
|
||||
.inputInfoBox01{float:none;}
|
||||
.inputInfoBox01 .inputInfo{width:100%;height:40px;margin-bottom:20px;clear:both;}
|
||||
.inputInfoBox01 .inputInfo label{float:left;line-height:40px;}
|
||||
.inputInfoBox01 .inputInfo label.label01{float:left;line-height:20px;width:66px;}
|
||||
.inputInfoBox01 .inputInfo label.label01a{clear:both;margin-bottom:17px;margin-top:17px;}
|
||||
.inputInfoBox01 .inputInfo label i{margin-left:5px;}
|
||||
.inputInfoBox01a{margin-right:0;}
|
||||
.inpu,.sel{height:40px;line-height:40px;}
|
||||
.inpu,.sel,.demand{width:89.5%;float:right;}
|
||||
.dialogBox .inpu{float:left;}
|
||||
.inpu01{width:70%;}
|
||||
.inpu02a{margin-top:17px;}
|
||||
#selectedCity{width:auto;}
|
||||
.error01{top:-17px;left:78px;}
|
||||
.user_btn,.line{width:100%;}
|
||||
.footerM{display:none;}
|
||||
}
|
||||
@media(max-device-width:414px){
|
||||
.inpu,.sel,.demand{width:79.5%;}
|
||||
.inpu01{width:45%;}
|
||||
}
|
||||
@media(max-device-width:375px){
|
||||
.inpu,.sel,.demand{width:77%;}
|
||||
.inpu01{width:40%;}
|
||||
}
|
||||
@media(max-device-width:320px){
|
||||
.inpu,.sel,.demand{width:73%;}
|
||||
.inpu01{width:30%;}
|
||||
.demand p span{width:80%;}
|
||||
}
|
||||
.img-box__img4 {
|
||||
background-color: black;
|
||||
width: 100%;
|
||||
height: 50px;
|
||||
object-fit: contain;
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
var retFlag = 1;
|
||||
</script>
|
||||
<script type="text/javascript" src="${staticUrl}/js/fangjiaNew.js?v=${version}"></script>
|
||||
<script src="${staticUrl}/js/fangjia/v5/main.js?v=${version}"></script>
|
||||
<script src="${staticUrl}/js/fangjia/v5/city.js?v=${version}"></script>
|
||||
|
||||
<script type="text/javascript" src="//g.alicdn.com/sj/lib/zepto/zepto.js" charset="utf-8"></script>
|
||||
|
||||
<script>
|
||||
|
||||
/* 关闭路由,以实现不同页面之间的相互跳转(仅开发环境需要) */
|
||||
Zepto = $
|
||||
$.config = {router: false}
|
||||
|
||||
</script>
|
||||
|
||||
<script src="${staticUrl}/js/fangjia/v5/light7.js?v=${version}"></script>
|
||||
<script src="${staticUrl}/js/fangjia/v5/cn.js?v=${version}"></script>
|
||||
|
||||
<script>
|
||||
var controllerName = '${controllerName}';
|
||||
var mid = '${mid!}'
|
||||
$(function(){
|
||||
if((navigator.userAgent.match(/(iPad|iPhone|iPod|Android|ios)/i))){
|
||||
$(".popup-about01 .tabs a").removeAttr("href");//去除城市页面跳转
|
||||
}
|
||||
})
|
||||
$(".user_btn").click(function(){
|
||||
if(!checkContactName()){
|
||||
scrollTo(0,0);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!checkCompany()){
|
||||
scrollTo(0,0);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!checkMobile2()){
|
||||
scrollTo(0,0);
|
||||
return false;
|
||||
}
|
||||
|
||||
/*if(!checkEmail()){
|
||||
scrollBy(0,-300);
|
||||
return false;
|
||||
}
|
||||
if(!checkWork()){
|
||||
scrollBy(0,-100);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!checkNature()){
|
||||
scrollBy(0,-200);
|
||||
return false;
|
||||
}
|
||||
if(mid=="1018"||mid=="1016"){
|
||||
if(!checkOtherInfo()){
|
||||
scrollBy(0,-100);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// var cls = $(".agreement i").attr("class");
|
||||
// if(cls.indexOf("checked")>-1){
|
||||
// $("#emailStatus").val(true);
|
||||
// }else{
|
||||
// $("#emailStatus").val(false);
|
||||
// }
|
||||
$("#_add_form").submit();
|
||||
});
|
||||
|
||||
function checkOtherNature(){
|
||||
var otherNature = $("#otherIndustryNature").val();
|
||||
if($.trim(otherNature) == ""){
|
||||
$("#otherIndustryNature").parent().find(".error:eq(0)").show();
|
||||
$("#otherIndustryNature").parent().find(".error:eq(0) span").text("其他行业性质不能为空");
|
||||
return false;
|
||||
}else{
|
||||
|
||||
$("#otherIndustryNature").parent().find(".error:eq(0)").hide();
|
||||
if(!checkIndustryNature()){
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkIndustryNature(){
|
||||
var otherWork = $("#otherWork2").val();
|
||||
if($.trim(otherWork) == ""){
|
||||
$("#otherIndustryNature").parent().find(".error:eq(1) span").text("其他工作职能不能为空");
|
||||
$("#otherIndustryNature").parent().find(".error:eq(1)").removeClass("dn")
|
||||
return false;
|
||||
}else{
|
||||
$("#otherIndustryNature").parent().find(".error:eq(1)").hide();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkWork(){
|
||||
var val = $("#nature").val();
|
||||
if(val!=""){
|
||||
if(val.indexOf("请填写")>0){
|
||||
return checkOtherNature();
|
||||
}else{
|
||||
var work = $(".duty:visible").find("select").val()
|
||||
if($.trim(work) == "请选择您的工作职能"){
|
||||
$(".duty:visible").find(".error span").text("工作职能不能为空");
|
||||
$(".duty:visible").find(".error").show();
|
||||
return false;
|
||||
}else if(work.indexOf("请填写")>-1){
|
||||
return checkOtherWork();
|
||||
}else{
|
||||
$("#work").val(work);
|
||||
$(".duty:visible").find(".error").text("");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
function checkOtherWork(){
|
||||
var otherWork = $("#otherWork").val();
|
||||
if($.trim(otherWork) == ""){
|
||||
$("#otherWork").parent().find(".error span").text("其他工作职能不能为空");
|
||||
$("#otherWork").parent().find(".error").show();
|
||||
return false;
|
||||
}else{
|
||||
$("#otherWork").parent().find(".error").text("");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkNature(){
|
||||
var val = $("#nature").val();
|
||||
if ($.trim(val) == "") {
|
||||
$("#nature").parent().find(".error span").text("行业性质不能为空");
|
||||
$("#nature").parent().find(".error").show();
|
||||
return false;
|
||||
}else{
|
||||
$("#nature").parent().find(".error").text("");
|
||||
|
||||
if(val.indexOf("请填写")>-1){
|
||||
return checkOtherNature();
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkMobile2(){
|
||||
var val = $("#mobile").val();
|
||||
if ($.trim(val) == "") {
|
||||
$("#mobile").parent().find(".error span").text("手机号码不能为空");
|
||||
$("#mobile").parent().find(".error").show();
|
||||
return false;
|
||||
}else if(!(/^1\d{10}$/.test(val))){
|
||||
$("#mobile").parent().find(".error span").text("手机号码格式错误");
|
||||
$("#mobile").parent().find(".error").show();
|
||||
return false;
|
||||
} else{
|
||||
$("#mobile").parent().find(".error").text("");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkContactName(){
|
||||
var val = $("#uname").val();
|
||||
if ($.trim(val) == "") {
|
||||
$("#uname").parent().find(".error span").text("您的姓名不能为空");
|
||||
$("#uname").parent().find(".error").show();
|
||||
return false;
|
||||
}else{
|
||||
$("#uname").parent().find(".error").hide();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkEmail(){
|
||||
var val = $("#email").val();
|
||||
if($.trim(val) == ""){
|
||||
$("#email").parent().find(".error span").text("电子邮箱不能为空");
|
||||
$("#email").parent().find(".error").show();
|
||||
return false;
|
||||
}else if (!/^([a-zA-Z0-9_-])+@([a-zA-Z0-9_-])+(.[a-zA-Z0-9_-])+/.test(val)) {
|
||||
$("#email").parent().find(".error span").text("电子邮箱格式错误");
|
||||
$("#email").parent().find(".error").show();
|
||||
return false;
|
||||
}else{
|
||||
$("#email").parent().find(".error").text("");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkCompany(){
|
||||
var val = $("#company").val();
|
||||
if ($.trim(val) === "") {
|
||||
$("#company").parent().find(".error span").text("公司名称不能为空");
|
||||
$("#company").parent().find(".error").show();
|
||||
return false;
|
||||
}else{
|
||||
$("#company").parent().find(".error").text("");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkOtherInfo(){
|
||||
var box = $(".demand p i.checked");
|
||||
if(box.length>0){
|
||||
$(".demand").parent().find(".error span").text("")
|
||||
$(".demand").parent().find(".error").hide();
|
||||
var selectOtherFlag = false;
|
||||
var content = []
|
||||
box.each(function(){
|
||||
var s = $(this).parent().find("span").text();
|
||||
if(s.indexOf("请填写")>0){
|
||||
selectOtherFlag = true;
|
||||
}
|
||||
content.push(s);
|
||||
})
|
||||
$("#business").val(content.join("_"));
|
||||
|
||||
if(selectOtherFlag){
|
||||
var otherBusiness = $("#otherBusiness").val();
|
||||
if ($.trim(otherBusiness) == "") {
|
||||
$("#otherBusiness").parent().find(".error span").text("其他业务需求不能为空");
|
||||
$("#otherBusiness").parent().find(".error").show();
|
||||
return false;
|
||||
}else{
|
||||
$("#otherBusiness").parent().find(".error").text("");
|
||||
}
|
||||
}
|
||||
}else{
|
||||
$(".demand").parent().find(".error span").text("请至少勾选1项业务需求")
|
||||
$(".demand").parent().find(".error").show();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
/* 定位城市 */
|
||||
$("body").on("click", ".letter a", function () {
|
||||
var s = $(this).html();
|
||||
$(".content").scrollTop(0);
|
||||
$(".content").scrollTop($("#" + s + "1").offset().top - 66);
|
||||
});
|
||||
$(".content-block01").scroll(function () {
|
||||
$(".content-block01 .letter").css("position","absolute");
|
||||
$(".content-block01 .letter").css("top", 116 - $("#cityContainer").position().top);
|
||||
});
|
||||
|
||||
/* 获取城市 移动端 */
|
||||
$(".popup-about01 .cityGPS").on("click", function () {
|
||||
$("#selectedCity,#city").val($(this).find("p").text());
|
||||
//$(".adBoxApp01").show();
|
||||
});
|
||||
$("body").delegate(".popup-about01 a","click", function(){
|
||||
$("#selectedCity,#city").val($(this).text());
|
||||
//$(".adBoxApp01").show();
|
||||
});
|
||||
$(".popup-about01-mobile .cityGPS").on("click", function () {
|
||||
$("#selectedCity,#city").val($(this).find("p").text());
|
||||
//$(".adBoxApp01").show();
|
||||
});
|
||||
$("body").delegate(".popup-about01-mobile a","click", function(){
|
||||
$("#selectedCity,#city").val($(this).text());
|
||||
//$(".adBoxApp01").show();
|
||||
});
|
||||
$(".popup-about01 .icon-left").on("click", function () {
|
||||
//$(".adBoxApp01").show();
|
||||
});
|
||||
|
||||
</script>
|
||||
</#escape>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,86 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
|
||||
<title>咨询信息提交成功 - 地数 didata.ai</title>
|
||||
<meta name="keywords" content="">
|
||||
<meta name="description" content="">
|
||||
|
||||
<link href="${staticUrl}/style/fangjia/v5/common.css?v=${version}" rel="stylesheet">
|
||||
<link href="${staticUrl}/style/fangjia/v5/main.css?v=${version}" rel="stylesheet">
|
||||
<link href="${staticUrl}/style/fangjia/v5/user.css?v=${version}" rel="stylesheet">
|
||||
<script src="${staticUrl}/js/fangjia/v5/jquery-1.7.2.min.js?v=${version}"></script>
|
||||
|
||||
<script>
|
||||
if((navigator.userAgent.match(/(iPad|iPhone|iPod|Android|ios)/i))){
|
||||
document.write("<link href='${staticUrl}/style/fangjia/v5/mobile.css?v=${version}' rel='stylesheet'>");
|
||||
}
|
||||
</script>
|
||||
|
||||
<style>
|
||||
@media(max-device-width:768px){
|
||||
.main{padding-top:0;}
|
||||
.main_right{float:none;width:100%;}
|
||||
.user{width:90%;margin-top:15rem;}
|
||||
.footerM{display:none;}
|
||||
}
|
||||
.img-box__img4 {
|
||||
background-color: black;
|
||||
width: 100%;
|
||||
height: 50px;
|
||||
object-fit: contain;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<div class="header">
|
||||
<div style="background-color: black;">
|
||||
<div class="logo" style="padding-top:2px;">
|
||||
<img class="img-box__img4" src="http://www.didata.ai/upload/20220522/3324a05faa1a88d483572ec64ac48c82.png"></img>
|
||||
<#-- <a href="http://www.didata.ai/upload/20220522/3324a05faa1a88d483572ec64ac48c82.png"></a>-->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="main">
|
||||
<div class="main_right">
|
||||
<div class="user user03">
|
||||
<div class="content">
|
||||
<p class="success"><i></i>您的信息已提交成功</p>
|
||||
<p class="link" style="color:#333;">我们的专业数据顾问将会尽快与您取得联系,感谢您的咨询!</p>
|
||||
<p class="note">若您有任何问题,欢迎拨打客服电话:<a>400-188-3787</a></p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
@media(max-device-width:768px){
|
||||
.main{padding-top:0;}
|
||||
.main_right{float:none;width:100%;}
|
||||
.user{width:90%;margin-top:15rem;}
|
||||
.footerM{display:none;}
|
||||
}
|
||||
</style>
|
||||
|
||||
<script src="${staticUrl}/js/fangjia/v5/main.js?v=${version}"></script>
|
||||
|
||||
<script>
|
||||
|
||||
var url = '${backUrl!}';
|
||||
var downUrl = '${downUrl!}';
|
||||
|
||||
if(!downUrl){
|
||||
setTimeout("location.href='"+url+"'",3000);
|
||||
}else{
|
||||
setTimeout("location.href='"+url+"&downUrl="+downUrl+"'",3000);
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
+2
-1
@@ -22,7 +22,8 @@ public enum SalesLeadEnum {
|
||||
DOWN_APPLY("1016", "下载申请"),
|
||||
FDANG_DIAO_BAO_GAO("1017", "房产专业调查评估报告"),
|
||||
BUSINESS_ASK("1018", "业务咨询"),
|
||||
HOUSE_EVAL("1019", "房产评估");
|
||||
HOUSE_EVAL("1019", "房产评估"),
|
||||
DIDATA("1020", "地数平台");
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -83,6 +83,8 @@ public class Loudong implements MongoEntity {
|
||||
private Date modifyDate;
|
||||
/**户数*/
|
||||
private Integer households;
|
||||
/**一梯几户的户*/
|
||||
private Integer roomCount;
|
||||
/** 总建筑面积 */
|
||||
private Double totalAreas;
|
||||
/** 开盘时间 */
|
||||
|
||||
@@ -102,6 +102,7 @@ public class FjtEvaluation implements SqlEntity{
|
||||
|
||||
private String finishDateStr;//竣工时间
|
||||
private String useType;//房屋用途
|
||||
private String certificateAddress; //莉痐華硊
|
||||
|
||||
private Date downPdfDate;//下载报告时间
|
||||
|
||||
@@ -753,6 +754,16 @@ public class FjtEvaluation implements SqlEntity{
|
||||
}
|
||||
|
||||
|
||||
@Column(name = "certificate_address")
|
||||
public String getCertificateAddress() {
|
||||
return certificateAddress;
|
||||
}
|
||||
|
||||
public void setCertificateAddress(String certificateAddress) {
|
||||
this.certificateAddress = certificateAddress;
|
||||
}
|
||||
|
||||
|
||||
@Column(name = "down_pdf_date")
|
||||
public Date getDownPdfDate() {
|
||||
return downPdfDate;
|
||||
|
||||
+2
-2
@@ -233,8 +233,8 @@ public class DealPriceController {
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1);
|
||||
}
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
|
||||
+4
-4
@@ -434,8 +434,8 @@ public class HouseController {
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1);
|
||||
}
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add(district.getName());
|
||||
@@ -487,8 +487,8 @@ public class HouseController {
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1);
|
||||
}
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add(district.getName());
|
||||
|
||||
+8
-1
@@ -41,8 +41,15 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_COMMBUILDTYPE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat"
|
||||
}
|
||||
|
||||
|
||||
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
}
|
||||
|
||||
+1
-3
@@ -44,9 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
// if(["坑鹵","굇쑴","밤鹵","볼鹵","북례","�柏","柯소鏤","嶠볶","鮫갛","路헤"].contains(city)){
|
||||
// continue
|
||||
// }
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
|
||||
println city +" ĐĄÇřÁżŁş"+dists.size()
|
||||
|
||||
+11
-10
@@ -52,21 +52,20 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = CommonDatautils.findAllCity()//CcbCommonUtils.citys;
|
||||
def ccbCitys = CcbCommonUtils.citys;
|
||||
|
||||
// for(def city : citys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// println citys.unique().size()
|
||||
for(def city : citys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
|
||||
// ppeDists = getPpeGTE15Datas(city)
|
||||
// }
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
@@ -570,8 +569,10 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20220101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220518, 20220619);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+3
-1
@@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -377,7 +378,8 @@ class CcbCommonUtils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
|
||||
println "ԭʼ:"+dbsour.count()
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
|
||||
+3
-3
@@ -373,10 +373,10 @@ class DealPriceDataProcess {
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202204
|
||||
def month = 202205
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202205)
|
||||
def rets = findPeGteThirdParty(city, 202206)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
@@ -388,7 +388,7 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+182
-106
@@ -23,7 +23,7 @@ class ExportOracleData {
|
||||
def run() {
|
||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def new_dists = ["350604","350605","410307","410308","450181"]
|
||||
//不会变
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()
|
||||
@@ -37,9 +37,9 @@ class ExportOracleData {
|
||||
exportCommunityStaging()
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-04-19")
|
||||
exportBigdataDeal("2022-04-01")
|
||||
exportThirdPartyPrice2(lastMonth, new_dists)
|
||||
exportBigdataForsale("2022-05-19", new_dists)
|
||||
exportBigdataDeal("2022-05-01", new_dists)
|
||||
|
||||
|
||||
//计算结果相关
|
||||
@@ -293,12 +293,51 @@ class ExportOracleData {
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDeal(def dealDate) {
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
|
||||
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
|
||||
def exportBigdataDeal(def dealDate, def new_dists) {
|
||||
|
||||
def allRows = []
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
|
||||
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}"
|
||||
def rets1 = dealData(sql1)
|
||||
|
||||
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
|
||||
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj where distid in ${newstr}"
|
||||
def rets2 = dealData(sql2)
|
||||
allRows.addAll(rets1)
|
||||
allRows.addAll(rets2)
|
||||
}else{
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
|
||||
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
|
||||
allRows = dealData(sql)
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_DEAL_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def dealData(def sql){
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -369,21 +408,9 @@ class ExportOracleData {
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_DEAL_GJ 上传Ftp完成......"
|
||||
return allRows
|
||||
}
|
||||
|
||||
|
||||
def exportThirdPartyPrice() {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" +
|
||||
"TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" +
|
||||
@@ -433,8 +460,42 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
|
||||
def exportThirdPartyPrice2(price_date) {
|
||||
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} ";
|
||||
def exportThirdPartyPrice2(price_date, new_dists) {
|
||||
|
||||
def allRows = []
|
||||
if(new_dists) {
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
|
||||
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
|
||||
"price_date=${price_date} and distid not in ${newstr} ";
|
||||
def rets1 = getThirdPartyDatas(sql1)
|
||||
allRows.addAll(rets1)
|
||||
|
||||
def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
|
||||
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
|
||||
"distid in ${newstr} ";
|
||||
def rets2 = getThirdPartyDatas(sql2)
|
||||
allRows.addAll(rets2)
|
||||
}else{
|
||||
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} ";
|
||||
allRows = getThirdPartyDatas(sql)
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
}
|
||||
|
||||
def getThirdPartyDatas(def sql){
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -452,21 +513,9 @@ class ExportOracleData {
|
||||
data << r.UPDATETIME
|
||||
allRows << data
|
||||
})
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
return allRows;
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommunity(){
|
||||
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
|
||||
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100)
|
||||
@@ -521,7 +570,7 @@ class ExportOracleData {
|
||||
println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataForsale(publishDate){
|
||||
def exportBigdataForsale(publishDate, new_dists){
|
||||
//["北京","上海"]
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -535,84 +584,111 @@ class ExportOracleData {
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
|
||||
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
|
||||
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
|
||||
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
def index = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
index = index+1
|
||||
println city +" "+index
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.FORSALEID
|
||||
data << r.PROPCODE
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.UNITPRICE
|
||||
data << r.TOTALPRICE
|
||||
data << r.PAYTYPE
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.BUILDTYPE
|
||||
data << r.HOUSESTRU
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.IFELEVATOR
|
||||
data << r.ELEVATORHOUSE
|
||||
data << r.PUBLISHTIME
|
||||
data << r.EXPIRETIME
|
||||
data << r.IFFIRST
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.PUBLISHER
|
||||
data << r.PHONE
|
||||
data << ""
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.ATTACH
|
||||
allRows << data
|
||||
|
||||
if(allRows.size()%200000==0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
allRows.clear()
|
||||
}
|
||||
});
|
||||
def allRows = []
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
|
||||
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
|
||||
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
|
||||
"where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
|
||||
def rets1 = getHouseDatas(sql1)
|
||||
allRows.addAll(rets1)
|
||||
}else{
|
||||
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
|
||||
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
|
||||
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
|
||||
allRows = getHouseDatas(sql)
|
||||
}
|
||||
if(allRows.size()>0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
if(new_dists){//补充被修区域的数据
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
|
||||
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
|
||||
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " +
|
||||
"from bigdata_forsale_gj where distid in ${newstr}"
|
||||
def rets2 = getHouseDatas(sql2)
|
||||
if(rets2.size()>0){
|
||||
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def getHouseDatas(sql){
|
||||
def allRows = []
|
||||
def index = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
index = index+1
|
||||
println r.CITYNAME +" "+index
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.FORSALEID
|
||||
data << r.PROPCODE
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.UNITPRICE
|
||||
data << r.TOTALPRICE
|
||||
data << r.PAYTYPE
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.BUILDTYPE
|
||||
data << r.HOUSESTRU
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.IFELEVATOR
|
||||
data << r.ELEVATORHOUSE
|
||||
data << r.PUBLISHTIME
|
||||
data << r.EXPIRETIME
|
||||
data << r.IFFIRST
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.PUBLISHER
|
||||
data << r.PHONE
|
||||
data << ""
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.ATTACH
|
||||
allRows << data
|
||||
// if(allRows.size()%200000==0){
|
||||
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
// allRows.clear()
|
||||
// }
|
||||
});
|
||||
return allRows
|
||||
}
|
||||
|
||||
|
||||
def exportAvmEvaluationGJNB(yearMonth){
|
||||
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-05-1", "2022-06-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+16
-1
@@ -23,7 +23,8 @@ class LoadGuide {
|
||||
}
|
||||
|
||||
def run(){
|
||||
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
|
||||
// addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
|
||||
addPersonGuided("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
|
||||
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||
@@ -38,6 +39,20 @@ class LoadGuide {
|
||||
|
||||
}
|
||||
|
||||
def addName(filename){
|
||||
ExcelUtils.multiThreadingModify(filename, { d ->
|
||||
if(d['A']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString())
|
||||
if(dist){
|
||||
d['B'] = dist.city
|
||||
d['C'] = dist.region
|
||||
d['D'] = dist.name
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addPersonGuided(filename){
|
||||
ExcelUtils.modify(filename, { d ->
|
||||
def city = d['B']
|
||||
|
||||
+5
-1
@@ -34,6 +34,10 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
println city +" 分期量:"+rets.size()
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/COMMUNITY_STAGING_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat"
|
||||
}
|
||||
|
||||
def results =[]
|
||||
def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
|
||||
@@ -41,7 +45,7 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
results.addAll(rows)
|
||||
|
||||
//ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
+34
-20
@@ -30,11 +30,11 @@ import java.util.regex.*;
|
||||
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
@@ -45,8 +45,20 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
def url = "https://sh.anjuke.com"
|
||||
def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
println s
|
||||
|
||||
//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁")
|
||||
//for(def d : districts){
|
||||
// println d.name
|
||||
//}
|
||||
|
||||
//def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"])
|
||||
//for(def d : dists){
|
||||
// def o = CommonDatautils.findDistrictById(d._id.toString())
|
||||
// println o._id.toString() + " "+o.name
|
||||
//}
|
||||
|
||||
//def citys = CcbCommonUtils.citys;
|
||||
//def rows = []
|
||||
@@ -236,16 +248,16 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
def rows = findTradePpeData(sqlInstance236)
|
||||
// rows.unique()
|
||||
def map=findTradeDetail(sqlInstance236,null)
|
||||
def datas=[]
|
||||
map.each{key,row->
|
||||
if(!rows.contains(key)){
|
||||
datas<<row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||
//def rows = findTradePpeData(sqlInstance236)
|
||||
//// rows.unique()
|
||||
//def map=findTradeDetail(sqlInstance236,null)
|
||||
//def datas=[]
|
||||
//map.each{key,row->
|
||||
// if(!rows.contains(key)){
|
||||
// datas<<row
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def datas=[]
|
||||
@@ -314,15 +326,17 @@ ExcelUtils.write("D:\\
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
//// def region = d['B']
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
//// if(newRegion) {
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
@@ -331,7 +345,7 @@ ExcelUtils.write("D:\\
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202202
|
||||
def endDate = 202204
|
||||
def sDate= 202205//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202203
|
||||
def endDate = 202205
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+6
-6
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","3月总量","4月总量","5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-02-01"
|
||||
def date = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
def newDate = "2022-05-01"
|
||||
def startDate = "2022-03-01"
|
||||
def date = "2022-04-01"
|
||||
def endDate = "2022-05-01"
|
||||
def newDate = "2022-06-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-05成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+194
-5
@@ -16,6 +16,7 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -27,9 +28,20 @@ class CcbDistrictToDB {
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.meituanRun()
|
||||
task.meituanSqrentRun()
|
||||
// task.meituanSqrentRun()
|
||||
// task.updateDistrict()
|
||||
// task.updateDongguangRegion()
|
||||
task.houseDistToDB()
|
||||
// task.sendCalPrice()
|
||||
}
|
||||
|
||||
def sendCalPrice(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","estate_type2"])
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-27")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-06-14")
|
||||
for(def data : dists){
|
||||
addPrice(data, startDate, endDate)
|
||||
}
|
||||
}
|
||||
|
||||
def updateDongguangRegion(){
|
||||
@@ -80,6 +92,160 @@ class CcbDistrictToDB {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def houseDistToDB(){
|
||||
def index = 0
|
||||
def addIndex = 0
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
ExcelUtils.modify("D:\\数据\\建行\\二期\\新增小区_2155.xlsx", { d ->
|
||||
index++
|
||||
if(d['A']){
|
||||
def city = d['A']
|
||||
def mapInfo = getStreetInfo(d['L']+","+d['M'])
|
||||
def newRegion = null
|
||||
if(mapInfo){
|
||||
if(d['A']=="东莞"){
|
||||
newRegion = mapInfo.township
|
||||
}else{
|
||||
newRegion = mapInfo.district
|
||||
}
|
||||
}
|
||||
|
||||
newRegion = RegionData.formatRegion(city, newRegion)
|
||||
|
||||
if(newRegion && newRegion.contains(d['B'])){
|
||||
def name = nameFileter(d['E'])
|
||||
def alias = []
|
||||
if(d['C']!=name){
|
||||
alias << d['C']
|
||||
}
|
||||
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name){
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['K']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def formatAddress = null
|
||||
if (d['J']) {
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['J'])
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
|
||||
if (d['M']) {
|
||||
glat = d['M'].toString().toDouble() * 10000000000L
|
||||
glng = d['L'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def key = city + "##" + newRegion + "##" + name
|
||||
if (name && !keys.contains(key)) {
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if (!dist) {
|
||||
addIndex++
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat", 0)
|
||||
data.put("status", 0)
|
||||
data.put("cat", "district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region", newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2", estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(alias){
|
||||
data.put("alias", alias)
|
||||
}
|
||||
if (glat) {
|
||||
data.put("lat", glat)
|
||||
data.put("lng", glng)
|
||||
data.put("lat2", blat)
|
||||
data.put("lng2", blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map", markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println addIndex + " " + data
|
||||
// def stop = false
|
||||
// while (!stop) {
|
||||
// if (data._id.toString()) {
|
||||
// addPrice(data)
|
||||
// stop = true;
|
||||
// }
|
||||
// }
|
||||
}else{
|
||||
if(alias){
|
||||
def dbAlias = dist.alias
|
||||
if(!dbAlias){
|
||||
def data = new BasicDBObject();
|
||||
data.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data))
|
||||
}else{
|
||||
for(def n : alias){
|
||||
if(!dbAlias.contains(n)){
|
||||
dbAlias << n
|
||||
}
|
||||
}
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("alias", dbAlias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data))
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(alias){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if (dist) {
|
||||
def dbAlias = dist.alias
|
||||
if(!dbAlias){
|
||||
def data = new BasicDBObject();
|
||||
data.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data))
|
||||
}else{
|
||||
for(def n : alias){
|
||||
if(!dbAlias.contains(n)){
|
||||
dbAlias << n
|
||||
}
|
||||
}
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("alias", dbAlias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C'] +" 经纬度所在区:"+newRegion
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def meituanSqrentRun(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\Êý¾Ý\\ÃÀÍÅ\\ÃÀÍÅ´ýÈë¿â.xlsx", { d ->
|
||||
@@ -228,11 +394,8 @@ class CcbDistrictToDB {
|
||||
}
|
||||
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
|
||||
def addPrice(dist, startDate, endDate){
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
@@ -521,4 +684,30 @@ class CcbDistrictToDB {
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
def getLocationInfo(def city, def address){
|
||||
def gloJson = GaoDeApiUtils.geo(city, address)
|
||||
if(gloJson) {
|
||||
def glocation = null
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
glocation = geocodes.get(0).location
|
||||
}
|
||||
return glocation
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getStreetInfo(def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
return addressComponent;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+7
-5
@@ -17,11 +17,11 @@ class DistrictMarge {
|
||||
//task.run()
|
||||
// task.repeatAddress()
|
||||
// task.removeDist()
|
||||
task.updateAddress()
|
||||
// task.addMappingAlias()
|
||||
// task.updateAddress()
|
||||
task.addMappingAlias()
|
||||
}
|
||||
def updateAddress(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址_0608.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B'].trim()
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
@@ -41,7 +41,7 @@ class DistrictMarge {
|
||||
}
|
||||
|
||||
def removeDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
|
||||
ExcelUtils.modify("D:\\数据\\建行\\删除小区数据_245_0608.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
@@ -52,6 +52,8 @@ class DistrictMarge {
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+d['C']+" 不存在"
|
||||
}
|
||||
}else{
|
||||
println d['A'] + " ¸ñʽ»¯Ê§°Ü "+d['B']
|
||||
@@ -62,7 +64,7 @@ class DistrictMarge {
|
||||
|
||||
def addMappingAlias(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名_0608.xlsx", { d ->
|
||||
index ++
|
||||
if (d['A']) {
|
||||
def region = d['B'].trim()
|
||||
|
||||
+98
-76
@@ -25,8 +25,9 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||
//按天导(每天一个文件夹)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"))
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
||||
|
||||
if(curDate.toString().endsWith("10")){
|
||||
exportBigdataCommunityGJRH()
|
||||
@@ -459,7 +460,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataDealFJW2(curDate, dealdate){
|
||||
def exportBigdataDealFJW2(curDate, dealdate, new_dists){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
@@ -472,86 +473,107 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def allRows = []
|
||||
if(curDate.toString().endsWith("28")){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
|
||||
"where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" +
|
||||
" distid not in ${newstr} ";//CITYID='${cityCode}' and
|
||||
def rets1 = createDeal(sql1)
|
||||
|
||||
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
|
||||
" where distid in ${newstr}"
|
||||
def rets2 = createDeal(sql2)
|
||||
allRows.addAll(rets1)
|
||||
allRows.addAll(rets2)
|
||||
}else{
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
allRows = createDeal(sql)
|
||||
}
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
def createDeal(sql){
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
return allRows;
|
||||
}
|
||||
|
||||
def exportAllBigdataDealFJW(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
public class GZIPUtils {
|
||||
|
||||
public static final String GZIP_ENCODE_UTF_8 = "UTF-8";
|
||||
|
||||
public static final String GZIP_ENCODE_ISO_8859_1 = "ISO-8859-1";
|
||||
|
||||
public static byte[] compress(String str, String encoding) {
|
||||
|
||||
if (str == null || str.length() == 0) {
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
|
||||
GZIPOutputStream gzip;
|
||||
|
||||
try {
|
||||
|
||||
gzip = new GZIPOutputStream(out);
|
||||
|
||||
gzip.write(str.getBytes(encoding));
|
||||
|
||||
gzip.close();
|
||||
|
||||
} catch ( Exception e) {
|
||||
|
||||
e.printStackTrace();
|
||||
|
||||
}
|
||||
|
||||
return out.toByteArray();
|
||||
|
||||
}
|
||||
|
||||
public static byte[] compress(String str) throws IOException {
|
||||
|
||||
return compress(str, GZIP_ENCODE_UTF_8);
|
||||
|
||||
}
|
||||
|
||||
public static byte[] uncompress(byte[] bytes) {
|
||||
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
|
||||
ByteArrayInputStream ins = new ByteArrayInputStream(bytes);
|
||||
|
||||
try {
|
||||
|
||||
GZIPInputStream ungzip = new GZIPInputStream(ins);
|
||||
|
||||
byte[] buffer = new byte[256];
|
||||
|
||||
int n;
|
||||
|
||||
while ((n = ungzip.read(buffer)) >= 0) {
|
||||
|
||||
out.write(buffer, 0, n);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
e.printStackTrace();
|
||||
|
||||
}
|
||||
|
||||
return out.toByteArray();
|
||||
|
||||
}
|
||||
|
||||
public static String uncompressToString(byte[] bytes, String encoding) {
|
||||
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
|
||||
ByteArrayInputStream ins = new ByteArrayInputStream(bytes);
|
||||
|
||||
try {
|
||||
|
||||
GZIPInputStream ungzip = new GZIPInputStream(ins);
|
||||
|
||||
byte[] buffer = new byte[256];
|
||||
|
||||
int n;
|
||||
|
||||
while ((n = ungzip.read(buffer)) >= 0) {
|
||||
|
||||
out.write(buffer, 0, n);
|
||||
|
||||
}
|
||||
|
||||
return out.toString(encoding);
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
e.printStackTrace();
|
||||
|
||||
}
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
public static String uncompressToString(byte[] bytes) {
|
||||
|
||||
return uncompressToString(bytes, GZIP_ENCODE_UTF_8);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 字符串转化成为16进制字符串
|
||||
* @param s
|
||||
* @return
|
||||
*/
|
||||
public static String strTo16(String s) {
|
||||
String str = "";
|
||||
for (int i = 0; i < s.length(); i++) {
|
||||
int ch = (int) s.charAt(i);
|
||||
String s4 = Integer.toHexString(ch);
|
||||
str = str + s4;
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
|
||||
String s = "123";
|
||||
println "result: "+strTo16(s)
|
||||
|
||||
System.out.println("字符串长度:"+s.length());
|
||||
|
||||
println compress(s)
|
||||
|
||||
System.out.println("解压后:"+uncompress(compress(s)).length);
|
||||
|
||||
System.out.println("解压字符串后::"+uncompressToString(compress(s)).length());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+429
@@ -0,0 +1,429 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.bytebuddy.description.type.TypeDescription
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
|
||||
class MeituanStat {
|
||||
|
||||
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
private static DBCollection on_line = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
|
||||
|
||||
def arroundColl = CrawalDBUtil.getArroundTempCol();
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new MeituanStat()
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def hexStr = "\\x1f\\x8b\\x08\\x00\\x17U\\xa4b\\x02\\xff342\\x06\\x00\\xd2cH\\x88\\x03\\x00\\x00\\x00"
|
||||
println URLDecoder.decode(hexStr.replaceAll("\\\\x", "%"), StandardCharsets.UTF_8.name())
|
||||
|
||||
println URLDecoder.decode(hexStr.replaceAll("\\\\x", "%"), "utf-8");
|
||||
// def s = com.fangjia.core.util.DataUtil.zip("123")
|
||||
// String str = new String(s, "utf-8");
|
||||
// def r = com.fangjia.core.util.DataUtil.unzips(s)
|
||||
// println s
|
||||
// println str
|
||||
// println r
|
||||
// def citys = ["北京","上海","保定","成都","广州","杭州","南京","深圳","天津","武汉","西安","重庆","吉林","常州","大连","东莞","佛山","福州","贵阳","哈尔滨","合肥","济南","昆明","南宁","宁波","青岛","厦门","沈阳","石家庄","苏州","太原","温州","无锡","长春","长沙","郑州","鞍山","包头","大庆","大同","海口","邯郸","衡阳","呼和浩特","湖州","淮安","惠州","济宁","嘉兴","江门","金华","荆州","兰州","连云港","临沂","柳州","洛阳","绵阳","南昌","南通","南阳","莆田","秦皇岛","泉州","三亚","汕头","绍兴","台州","泰州","唐山","威海","潍坊","乌鲁木齐","芜湖","襄阳","徐州","烟台","盐城","扬州","宜昌","银川","岳阳","湛江","镇江","中山","珠海","株洲","淄博"]
|
||||
// def path = "D:\\数据\\美团\\美团88个城市统计.xlsx"
|
||||
// def rets =[]
|
||||
// rets << ["城市", "楼盘总量"]
|
||||
// for(def c : citys){
|
||||
// def count = CommonDatautils.findAllDistrictCount("district", c)
|
||||
// rets << [c, count]
|
||||
// }
|
||||
// ExcelUtils.write(path, rets, "总量")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def citys = ["北京","上海","保定","成都","广州","杭州","南京","深圳","天津","武汉","西安","重庆","吉林","常州","大连","东莞","佛山","福州","贵阳","哈尔滨","合肥","济南","昆明","南宁","宁波","青岛","厦门","沈阳","石家庄","苏州","太原","温州","无锡","长春","长沙","郑州","鞍山","包头","大庆","大同","海口","邯郸","衡阳","呼和浩特","湖州","淮安","惠州","济宁","嘉兴","江门","金华","荆州","兰州","连云港","临沂","柳州","洛阳","绵阳","南昌","南通","南阳","莆田","秦皇岛","泉州","三亚","汕头","绍兴","台州","泰州","唐山","威海","潍坊","乌鲁木齐","芜湖","襄阳","徐州","烟台","盐城","扬州","宜昌","银川","岳阳","湛江","镇江","中山","珠海","株洲","淄博"]
|
||||
def path = "D:\\数据\\美团\\美团88个城市统计.xlsx"
|
||||
def rets =[]
|
||||
rets << ["城市","ID","区域","小区名","小区别名","地址","地址别名","物业类型","建筑类型","经度","纬度","板块","开发商","物业公司",
|
||||
"环线","商圈","楼栋总数","户数","物业费","占地面积","建筑面积","容积率","绿化率","停车位","停车位(描述)","产权年限",
|
||||
"产权性质","建成年代","开工时间","开盘时间","竣工时间","交房时间","装修情况","建筑结构","周边配套","交通状况","相关信息",
|
||||
"描述","售楼地址","售楼电话","折扣信息","预售许可证","房产评级","房产均价","租金均价"]
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["lat","lng","loudong_count","mobile","car_site","fitment","relate_info","description","sell_address","discount","facilities","traffic","region","name"])
|
||||
def rMap = statFacilices(dists, c)
|
||||
def obj = findFieldCov(c)
|
||||
def rentCount = CommonDatautils.findSqmetreRents(c, DateUtils.parseFromTightByPattern("2022-05-31","yyyy-MM-dd"))
|
||||
def block = obj.district_detail.block?obj.district_detail.block*100:''
|
||||
def alias = obj.district_detail.alias?obj.district_detail.alias*100:''
|
||||
def address= obj.district_detail.address?obj.district_detail.address*100:''
|
||||
def addrAlias = obj.district_detail.addrAlias?obj.district_detail.addrAlias*100:''
|
||||
def estateType2 = obj.district_detail.estateType2?obj.district_detail.estateType2*100:''
|
||||
def buildType = obj.district_detail.buildType?obj.district_detail.buildType*100:''
|
||||
def lng = obj.district_detail.lng?obj.district_detail.lng*100:''
|
||||
def developer = obj.district_detail.developer?obj.district_detail.developer*100:''
|
||||
def estateCompany = obj.district_detail.estateCompany?obj.district_detail.estateCompany*100:''
|
||||
def loopLineInfo = obj.district_detail.loopLineInfo?obj.district_detail.loopLineInfo*100:''
|
||||
def businessQuarter = obj.district_detail.businessQuarter?obj.district_detail.businessQuarter*100:''
|
||||
def loudong = ''
|
||||
def loudongcount = getCount(dists,"loudong_count")
|
||||
if(obj.existing_district_count && loudongcount){
|
||||
def pre = loudongcount*100/obj.existing_district_count
|
||||
loudong = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
|
||||
def householdCount = obj.district_detail.householdCount?obj.district_detail.householdCount*100:''
|
||||
def estateCharge = obj.district_detail.estateCharge?obj.district_detail.estateCharge*100:''
|
||||
def allArea = obj.district_detail.allArea?obj.district_detail.allArea*100:''
|
||||
def buildArea = obj.district_detail.buildArea?obj.district_detail.buildArea*100:''
|
||||
def cubagePercent = obj.district_detail.cubagePercent?obj.district_detail.cubagePercent*100:''
|
||||
def virescencePercent = obj.district_detail.virescencePercent?obj.district_detail.virescencePercent*100:''
|
||||
def carSite = obj.district_detail.carSite?obj.district_detail.carSite*100:''
|
||||
def carSiteStr = ''
|
||||
def car_site_count = getCount(dists,"car_site")
|
||||
if(obj.existing_district_count && car_site_count){
|
||||
def pre = car_site_count*100/obj.existing_district_count
|
||||
carSiteStr = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def propertyLife = obj.district_detail.propertyLife?obj.district_detail.propertyLife*100:''
|
||||
def houseProperty = obj.district_detail.houseProperty?obj.district_detail.houseProperty*100:''
|
||||
def buildDate = obj.district_detail.buildDate?obj.district_detail.buildDate*100:''
|
||||
def startBuildingDate = obj.district_detail.startBuildingDate?obj.district_detail.startBuildingDate*100:''
|
||||
def openDate = obj.district_detail.openDate?obj.district_detail.openDate*100:''
|
||||
def finishBuildingDate = obj.district_detail.finishBuildingDate?obj.district_detail.finishBuildingDate*100:''
|
||||
def liveDate = obj.district_detail.liveDate?obj.district_detail.liveDate*100:''
|
||||
def fitment = ''
|
||||
def fitment_count = getCount(dists,"fitment")
|
||||
if(obj.existing_district_count && fitment_count){
|
||||
def pre = fitment_count*100/obj.existing_district_count
|
||||
fitment = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def buildStructure = obj.district_detail.buildStructure?obj.district_detail.buildStructure*100:''
|
||||
def arround= ''
|
||||
def facilities_count = getCount(dists,"facilities")
|
||||
if(obj.existing_district_count && facilities_count){
|
||||
def pre = facilities_count*100/obj.existing_district_count
|
||||
arround = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
|
||||
def traffic = ''
|
||||
def traffic_count = getCount(dists,"traffic")
|
||||
if(obj.existing_district_count && traffic_count){
|
||||
def pre = traffic_count*100/obj.existing_district_count
|
||||
traffic = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
|
||||
def relateInfo = ''
|
||||
def relate_info_count = getCount(dists,"relate_info")
|
||||
if(obj.existing_district_count && relate_info_count){
|
||||
def pre = relate_info_count*100/obj.existing_district_count
|
||||
relateInfo = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def description = ''
|
||||
def description_count = getCount(dists,"description")
|
||||
if(obj.existing_district_count && description_count){
|
||||
def pre = description_count*100/obj.existing_district_count
|
||||
description = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def sellAddress = ''
|
||||
def sell_address_count = getCount(dists,"sell_address")
|
||||
if(obj.existing_district_count && sell_address_count){
|
||||
def pre = sell_address_count*100/obj.existing_district_count
|
||||
sellAddress = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def mobile =''
|
||||
def mobile_count = getCount(dists,"mobile")
|
||||
if(obj.existing_district_count && mobile_count){
|
||||
def pre = mobile_count*100/obj.existing_district_count
|
||||
mobile = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def discount = ''
|
||||
def dis_count = getCount(dists,"discount")
|
||||
if(obj.existing_district_count && dis_count){
|
||||
def pre = dis_count*100/obj.existing_district_count
|
||||
discount = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
def licence = obj.district_detail.licence?obj.district_detail.licence*100:''
|
||||
def price = obj.price_coverage?obj.price_coverage*100:''
|
||||
def level = price
|
||||
def rentPrie = ''
|
||||
if(obj.existing_district_count && rentCount){
|
||||
def pre = rentCount*100/obj.existing_district_count
|
||||
rentPrie = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
|
||||
def db_fc = ''
|
||||
def db_bus = ''
|
||||
if(rMap){
|
||||
def count = obj.existing_district_count
|
||||
if(rMap.bus){
|
||||
def pre = rMap.bus*100/count
|
||||
db_bus = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
if(rMap.fc){
|
||||
def pre = rMap.fc*100/count
|
||||
db_fc = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
}
|
||||
rets << [c, 100,100,100,alias,address,addrAlias,estateType2,buildType,lng,lng,block,developer,estateCompany,
|
||||
loopLineInfo,businessQuarter,loudong,householdCount,estateCharge,allArea,buildArea,cubagePercent,virescencePercent,carSite,carSiteStr,propertyLife,
|
||||
houseProperty,buildDate,startBuildingDate,openDate,finishBuildingDate,liveDate,fitment,buildStructure,arround,traffic,relateInfo,
|
||||
description,sellAddress,mobile,discount,licence,level,price,rentPrie, db_bus, db_fc]
|
||||
println c +" 完成"
|
||||
}
|
||||
ExcelUtils.write(path, rets, "覆盖率")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.modify("D:\\数据\\美团\\美团88个城市统计.xlsx",{d->
|
||||
def dists = CommonDatautils.findAllDistrictByCity(d['A'], "district", ["name","lat","lng"])
|
||||
def rMap = statFacilices(dists)
|
||||
if(rMap){
|
||||
def count = dists.size()
|
||||
if(rMap.bus){
|
||||
def pre = rMap.bus*100/count
|
||||
d['AT'] = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
if(rMap.fc){
|
||||
def pre = rMap.fc*100/count
|
||||
d['AU'] = DataHandleUtils.getDoubleRound(2, pre)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findFieldCov(c){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", c)
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern("2022-06-05", "yyyy-MM-dd"))
|
||||
def obj = CITY_INFO_CONN.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def getCount(dists,field){
|
||||
def count = 0
|
||||
for(def d : dists){
|
||||
def val = d.get(field)
|
||||
if(val && !val.toString().contains("暂无")){
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
def statFacilices(dists){
|
||||
// def distMap = findArroundFaciltices(city)
|
||||
def bus_count = 0
|
||||
def fc_count = 0
|
||||
// for(def d : dists){
|
||||
// def lng = d.lng/10000000000.0
|
||||
// def lat = d.lat/10000000000.0
|
||||
// def fcCount = findFacByEs(lng, lat, ["160100","060400","141200","141201","141202","141203","090100","160300"])
|
||||
// def busCount = findFacByEs(lng, lat, ["150700", "150500"])
|
||||
// println fcCount
|
||||
// println busCount
|
||||
// }
|
||||
|
||||
def total = new AtomicLong()
|
||||
def busCount = new AtomicLong()
|
||||
def fcCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
total.addAndGet(1)
|
||||
def lng = d.lng/10000000000.0
|
||||
def lat = d.lat/10000000000.0
|
||||
def fc = findFacByEs(lng, lat, ["160100","060400","141200","141201","141202","141203","090100","160300"])
|
||||
def bus = findFacByEs(lng, lat, ["150700", "150500"])
|
||||
if(bus && busCount>0){
|
||||
busCount.addAndGet(1)
|
||||
}
|
||||
if(fc && fc>0){
|
||||
fcCount.addAndGet(1)
|
||||
}
|
||||
println total.get()+" "+ d.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
def rMap = [:]
|
||||
rMap.bus = busCount.get()
|
||||
rMap.fc = fcCount.get()
|
||||
return rMap
|
||||
}
|
||||
|
||||
def findFacByEs(lng, lat, codes){
|
||||
def must = []
|
||||
def term1 = ["terms":["typecode":codes]]
|
||||
|
||||
must << term1
|
||||
def params = ["query":["bool":["must":must,"filter":["geo_distance": [
|
||||
"distance": "1km","location": ["lat": lat,"lon": lng]]]]]]
|
||||
def s = httpSaveData(params, "http://192.168.0.206:9200/amap_poi/_search");
|
||||
def total = 0
|
||||
if(s){
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
total = hits.total.value
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findArroundFaciltices(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("matched_city", city)
|
||||
def dbcursor = arroundColl.find(q)
|
||||
def distMap = [:]
|
||||
for(def d : dbcursor){
|
||||
|
||||
def key = d.matched_region+"#"+d.matched_name
|
||||
def dMap = distMap.get(key)
|
||||
if(!dMap){
|
||||
dMap = [:]
|
||||
distMap.put(d.region+"#"+d.district_name, dMap)
|
||||
}
|
||||
|
||||
def bus = 0
|
||||
def fc = 0
|
||||
if(d.type){
|
||||
if(d.type.contains("公交") || d.type.contains("地铁")){
|
||||
bus = 1
|
||||
fc = 1
|
||||
}else{
|
||||
fc = 1
|
||||
}
|
||||
}
|
||||
// if((d.bus && d.bus.size()>0) || (d.school && d.school.size()>0) || (d.bank && d.bank.size()>0) ||(d.supermarket && d.supermarket.size()>0) || (d.hospital && d.hospital.size()>0) || (d.atm && d.atm.size()>0)|| (d.university && d.university.size()>0)|| (d.middle_school && d.middle_school.size()>0) || (d.primary_school && d.primary_school.size()>0)|| (d.kindergarten && d.kindergarten.size()>0)){
|
||||
// fc = 1
|
||||
// }
|
||||
dMap.bus = bus
|
||||
dMap.fc = fc
|
||||
}
|
||||
return distMap
|
||||
}
|
||||
|
||||
def findAllFaciltices(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def dbcursor = on_line.find(q)
|
||||
def distMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def bus = 0
|
||||
def fc = 0
|
||||
if(d.bus && d.bus.size()>0){
|
||||
bus = 1
|
||||
}
|
||||
if((d.bus && d.bus.size()>0) || (d.school && d.school.size()>0) || (d.bank && d.bank.size()>0) ||(d.supermarket && d.supermarket.size()>0) || (d.hospital && d.hospital.size()>0) || (d.atm && d.atm.size()>0)|| (d.university && d.university.size()>0)|| (d.middle_school && d.middle_school.size()>0) || (d.primary_school && d.primary_school.size()>0)|| (d.kindergarten && d.kindergarten.size()>0)){
|
||||
fc = 1
|
||||
}
|
||||
def dMap = [:]
|
||||
dMap.bus = bus
|
||||
dMap.fc = 1
|
||||
distMap.put(d.region+"#"+d.district_name, dMap)
|
||||
}
|
||||
return distMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
def httpSaveData(def params, def url){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz")
|
||||
return doPost(url,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true);
|
||||
conn.setUseCaches(false);
|
||||
conn.setConnectTimeout(5000);
|
||||
conn.setReadTimeout(5000);
|
||||
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
conn.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
} else {
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=" + CHARSET);
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||
param = JacksonUtil.beanToJson(params)
|
||||
}
|
||||
|
||||
conn.connect();
|
||||
|
||||
//输出流
|
||||
if (StringUtils.isNotEmpty(param)) {
|
||||
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||
mWriter.write(param);
|
||||
mWriter.flush();
|
||||
mWriter.close();
|
||||
}
|
||||
if (conn.getHeaderField("Content-Encoding") != null
|
||||
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||
} else {
|
||||
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||
}
|
||||
char[] bf = new char[1024];
|
||||
int len;
|
||||
while ((len = inStream.read(bf)) != -1) {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
}
|
||||
return buffer.toString();
|
||||
}
|
||||
}
|
||||
+60
-28
@@ -239,6 +239,27 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findRemoveDistricts(def city, def cat, def fields, def m_date){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status", 1);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(m_date)))
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取城市中小区的经纬度
|
||||
* @param city
|
||||
@@ -834,39 +855,42 @@ class CommonDatautils {
|
||||
if(!id){
|
||||
return null;
|
||||
}
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("alias", 1);
|
||||
keys.put("addr_alias", 1);
|
||||
keys.put("city", 1);
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("estate_type2", 1);
|
||||
keys.put("address", 1);
|
||||
keys.put("build_structure", 1);
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("lat2", 1);
|
||||
keys.put("lng2", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("alias", 1);
|
||||
keys.put("addr_alias", 1);
|
||||
keys.put("city", 1);
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("estate_type2", 1);
|
||||
keys.put("address", 1);
|
||||
keys.put("build_structure", 1);
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("lat2", 1);
|
||||
keys.put("lng2", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error_id:"+id
|
||||
e.printStackTrace()
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error_id:"+id
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -3200,6 +3224,14 @@ class CommonDatautils {
|
||||
return dbo;
|
||||
}
|
||||
|
||||
static def findSqmetreRents(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def count = SQMETRE_RENT_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
static def findNewSqmetreRent(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
+2
-1
@@ -34,7 +34,8 @@ class FjtResponseTimeAnaly extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run();
|
||||
//run();
|
||||
jms.sendMail("gaojuan@fangjia.com", "测试发邮件", "收到了?")
|
||||
}
|
||||
|
||||
|
||||
|
||||
+6
-3
@@ -68,14 +68,17 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def createDatas(def city){
|
||||
def result = []
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
districts.addAll(removeDists)
|
||||
|
||||
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
|
||||
def fzAddrMap = getFzAddress();
|
||||
for(def rMap : districts){
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
if(dbo){
|
||||
//价格没有也提供
|
||||
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
// if(dbo){
|
||||
def row = []
|
||||
row << rMap.get("_id").toString()
|
||||
row << rMap.city
|
||||
@@ -112,7 +115,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
}
|
||||
//row << processAlias(rMap.addr_alias)
|
||||
result << row
|
||||
}
|
||||
// }
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
+18
-30
@@ -88,6 +88,9 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def createDatas(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
districts.addAll(removeDists)
|
||||
|
||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||
def dataStr = DateUtils.add(endDateStr, -30)
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
|
||||
@@ -105,9 +108,10 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
for(def rMap : districts){
|
||||
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
def curPrice = 0
|
||||
if(dbo){
|
||||
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
||||
def curPrice = dbo.get("avg_price")
|
||||
curPrice = dbo.get("avg_price")
|
||||
if(lastMonthPrice && curPrice){
|
||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||
if(Math.abs(markup)>3){
|
||||
@@ -120,36 +124,20 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}
|
||||
|
||||
def m = []
|
||||
m << dbo.get("_id").toString()//¾ù¼ÛID
|
||||
m << rMap.get("_id").toString()//Â¥ÅÌID
|
||||
m << rMap.city
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << curPrice
|
||||
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
|
||||
// if(lastMonthPrice && dbo.get("avg_price")){
|
||||
// lastMonthPrice = lastMonthPrice.toInteger()
|
||||
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
|
||||
// if(Math.abs(markup)>10){
|
||||
// def random = 0
|
||||
// if(dbo.get("avg_price")-lastMonthPrice>=0){
|
||||
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
|
||||
// }else{
|
||||
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
|
||||
// }
|
||||
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
|
||||
// m << newPrice.toInteger()
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
def m = []
|
||||
if(curPrice){
|
||||
m << dbo.get("_id").toString()//¾ù¼ÛID
|
||||
}else{
|
||||
m << ""
|
||||
}
|
||||
m << rMap.get("_id").toString()//Â¥ÅÌID
|
||||
m << rMap.city
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << curPrice
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202205.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
@@ -136,8 +136,8 @@ def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
println s
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
//def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
//DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
|
||||
+5
@@ -143,9 +143,14 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("anjuke_esf_price")
|
||||
}
|
||||
|
||||
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("58tc_esf_price")
|
||||
}
|
||||
|
||||
if(!retMap.soufang_esf_price && d.soufang_esf_price){
|
||||
retMap.soufang_esf_price = d.soufang_esf_price
|
||||
}
|
||||
|
||||
@@ -165,7 +165,7 @@ class OpenApiUtils {
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
com.alibaba.fastjson.JSONObject result = null;
|
||||
|
||||
@@ -12,7 +12,8 @@ class OpenUserBatch {
|
||||
def task = new OpenUserBatch();
|
||||
// task.run();
|
||||
// task.test();
|
||||
task.addShenZhenFjtUser();
|
||||
// task.addShenZhenFjtUser();
|
||||
task.addFujianFjtUser()
|
||||
}
|
||||
|
||||
def test(){
|
||||
@@ -63,4 +64,23 @@ class OpenUserBatch {
|
||||
}
|
||||
}
|
||||
|
||||
def addFujianFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\福建省农行账号开通.xlsx",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252738 + i;
|
||||
String mobile = data['A']
|
||||
String pwd = '111111'
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '"+pwd+"', '12', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '100', NULL, NULL, '0', NULL, NULL, '100', '2022-07-10 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data['C']+"', '福建', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '蒋晓春')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'福州,厦门,泉州,莆田,宁德,三明,龙岩,南平,漳州','福州:,厦门:,泉州:,莆田:,宁德:,三明:,龙岩:,南平:,漳州:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -10,11 +10,11 @@ import java.text.SimpleDateFormat
|
||||
|
||||
class TestFjtLog {
|
||||
|
||||
public static final DBCollection COLL = MongoUtils.getCollectionByName("fangjiatong_log", "fjt_select_log", "192.168.0.42", 52002)
|
||||
public static final DBCollection COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
|
||||
static main(args) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat","芩華")
|
||||
// q.put("cat","芩華")
|
||||
def dbcursor = COLL.find(q).sort(new BasicDBObject("_id",-1))
|
||||
|
||||
def map = [:]
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package com.fangjia.script.gj.abuyun
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class AnjukeTest {
|
||||
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeTest()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = detail.find(q)
|
||||
for(def d : dbcursor){
|
||||
def url = d.jumpAction
|
||||
def s =ClaABuYun.getResponse(url)
|
||||
println(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.fangjia.script.gj.abuyun
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.Authenticator;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.PasswordAuthentication;
|
||||
import java.net.Proxy;
|
||||
import java.net.URL;
|
||||
|
||||
class ProxyAuthenticator extends Authenticator {
|
||||
private String user, password;
|
||||
|
||||
public ProxyAuthenticator(String user, String password) {
|
||||
this.user = user;
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(user, password.toCharArray());
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.abuyun.ClaABuYun
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
|
||||
class AnjukeAbuyunCrawl {
|
||||
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeAbuyunCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def url = "https://fz.anjuke.com/sale/?from=navigation"
|
||||
for(def i=2;i<50;i++){
|
||||
url ="https://fz.anjuke.com/sale/p${i}/?from=navigation"
|
||||
def doc = getABuYunDocument(url);
|
||||
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles!=null?eles.size():0
|
||||
if(eles){
|
||||
parser(eles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def parser(eles){
|
||||
def index = 0
|
||||
for(def e : eles){
|
||||
index ++
|
||||
def href = e.attr("href")
|
||||
def doc = getABuYunDocument(href);
|
||||
def newUrl = href.substring(0, href.indexOf("?"))
|
||||
sendMq(newUrl, doc)
|
||||
println "[ "+index +"] "+ newUrl
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getABuYunDocument(url){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
def s = ClaABuYun.getResponse(url)
|
||||
doc = Jsoup.parse(s);
|
||||
def els = JQueryUtils.select(".list-main .list-left .list", doc)
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".props-main", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
println "[${error}] error: "+url
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def sendMq(href, doc){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "sell_anjuke")
|
||||
json.put("url", href)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", doc.toString())
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
|
||||
}
|
||||
+347
-24
@@ -1,31 +1,239 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringEscapeUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import javax.script.Invocable
|
||||
import javax.script.ScriptEngine
|
||||
import javax.script.ScriptEngineManager
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AnjukeAppCrawl {
|
||||
|
||||
static def PREFIX_UNICODE = "\\u";
|
||||
public static final SHORT_DATE = Utils.getLastMonth();
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
public static def SOURCE = "갛앙와";
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
|
||||
// String line2 = StringEscapeUtils.unescapeJava(s);
|
||||
// println line2
|
||||
def task = new AnjukeAppCrawl();
|
||||
// task.autoRun()
|
||||
// task.run();
|
||||
// task.run2()
|
||||
|
||||
task.addThirdPrice()
|
||||
// task.saveToEs()
|
||||
// task.saveToEs1()
|
||||
|
||||
// task.groupByCity()
|
||||
// task.selectNoSave()
|
||||
// task.statCount()
|
||||
}
|
||||
def saveToEs1(){
|
||||
def lists = getAllAppDatas()
|
||||
anjukeToDB(lists)
|
||||
}
|
||||
|
||||
def statCount(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\anjuke\\cur_city_count.xlsx",{d->
|
||||
def city = d['B']
|
||||
if(!d['D']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("base.cityName",city)
|
||||
def count = detail.count(q)
|
||||
d['D'] = count
|
||||
println city +" "+count
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def selectNoSave(){
|
||||
def esData = findAllEsData()
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
if(!esData.contains(d.jumpAction)){
|
||||
println "没保存 "+ d.jumpAction
|
||||
}
|
||||
if(index%100000==0){
|
||||
println index
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findAllEsData(){
|
||||
def urls = new HashSet()
|
||||
ElasticUtils.executeSqlQuery("select url from community where source='安居客移动端'", {
|
||||
urls << it.url
|
||||
})
|
||||
return urls;
|
||||
}
|
||||
|
||||
def groupByCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def map = [:]
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def city = d.base.cityName
|
||||
def str = map.get(city)
|
||||
if(!str){
|
||||
map.put(city, d.base.longCityListName)
|
||||
}
|
||||
println index
|
||||
}
|
||||
|
||||
map.each {k,v->
|
||||
println k
|
||||
}
|
||||
}
|
||||
|
||||
def autoRun(){
|
||||
def rets = ExcelUtils.read("d:\\数据\\安居客城市区域编码.xlsx")
|
||||
def total= rets.size()
|
||||
def index =0
|
||||
for(def d : rets){
|
||||
index++
|
||||
println index+" "+ d['A']+" "+d['C']
|
||||
getDetailByRegion(d['A'], d['B'], d['D'], null)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def saveToEs(){
|
||||
def rets = ExcelUtils.read("d:\\数据\\安居客城市区域编码.xlsx",2,0)
|
||||
def citys = ["福州"]
|
||||
for(def d : rets){
|
||||
if(citys.contains(d['A'])){
|
||||
def lists = getAllAppDatas(d['A'])
|
||||
anjukeToDB(lists)
|
||||
citys << d['A']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def anjukeToDB(def lists){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(lists.size())
|
||||
for(def i=0;i<lists.size();i++){
|
||||
final def m = lists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def flag = FangjiaClientUtils.saveToEsDistrict(m)
|
||||
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def getAllAppDatas(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def list = []
|
||||
|
||||
for(def d : dbcursor){
|
||||
def map = [:]
|
||||
def base = d.base
|
||||
def extend = d.extend
|
||||
map.id = base.id
|
||||
map.city = base.cityName
|
||||
map.region = base.areaName
|
||||
map.name = base.name
|
||||
map.tradingArea = base.tradingAreaName
|
||||
map.address = base.address
|
||||
// map.alias": "string",
|
||||
map.avgPrice = d.priceInfo.price?d.priceInfo.price.toInteger():null
|
||||
map.buildArea = extend.totalArea?extend.totalArea.toDouble().toInteger():null
|
||||
map.buildType = base.buildTypeStr
|
||||
map.buildYear = base.completionTime
|
||||
map.greeningRate = extend.landscapingRatio?extend.landscapingRatio.toDouble():null
|
||||
map.households = extend.totalHouseHoldNum?extend.totalHouseHoldNum.toInteger():null
|
||||
if(base.lng){
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(base.lng.toString().toDouble(), base.lat.toString().toDouble())
|
||||
if(gmap){
|
||||
map.location = gmap.lng?gmap.lng+","+gmap.lat:''
|
||||
}
|
||||
}
|
||||
|
||||
map.propertyCompany = extend.propertyCompany
|
||||
map.propertyFee = extend.propertyMoney
|
||||
if(base.propertyTypes){
|
||||
map.propertyType = base.propertyTypes.join(',')
|
||||
}
|
||||
|
||||
map.rentNum = d.propInfo.rentNum?d.propInfo.rentNum.toInteger():null
|
||||
map.saleNum = d.propInfo.saleNum?d.propInfo.saleNum.toInteger():null
|
||||
map.source = "安居客移动端"
|
||||
map.url = d.jumpAction
|
||||
map.waterPowerSupply = extend.waterPowerSupply
|
||||
list << map
|
||||
if(list.size()/10000==0){
|
||||
println list.size()
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
def createRegionCode(){
|
||||
def url = "https://m.anjuke.com/cityList/"
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
def cityMap = getCityMap(url, driver)
|
||||
def rets = []
|
||||
rets << ["城市","城市id","区域","区域id"]
|
||||
def index = 0
|
||||
cityMap.each{ch_city,m->
|
||||
index++
|
||||
def regionDatas = getRegions(m.url+"community/?from=anjuke_home", driver)
|
||||
println "[${index}] "+ch_city+" "+regionDatas.size()
|
||||
regionDatas.each{k,v->
|
||||
|
||||
rets << [ch_city, m.city_id.toInteger(),v.info.name, v.info.value_str.toInteger()]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\安居客城市区域编码.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
@@ -46,6 +254,58 @@ class AnjukeAppCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def getRegions(url,driver){
|
||||
def doc = getDocument(url, driver)
|
||||
def start = doc.toString().indexOf("<script>")
|
||||
def end = doc.toString().indexOf(";</script>")
|
||||
if(start>0 && end>0 && end > start){
|
||||
def str = doc.toString().substring(start, end).replace("<script>","").replace("window.__NUXT__=","")
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
|
||||
def jsDatas = engine.eval(str.toString());
|
||||
def data = jsDatas.get("data")
|
||||
if(data){
|
||||
def dataArr = data.get("0")
|
||||
def regionDatas = dataArr.filter.regions
|
||||
return regionDatas;
|
||||
}
|
||||
}else{
|
||||
println url+" 无数据"
|
||||
}
|
||||
|
||||
return null
|
||||
// println doc.toString().indexOf("(function(")
|
||||
// println doc.toString().indexOf('){')
|
||||
// def str1 = doc.toString().substring(doc.toString().indexOf("(function("), doc.toString().indexOf('){')).replace("(function(","")
|
||||
// def str2 = doc.toString().substring(doc.toString().indexOf("pinyin=w;return")+15, doc.toString().lastIndexOf('}(')+1)
|
||||
// def str3 = doc.toString().substring(doc.toString().lastIndexOf("}(")+2, doc.toString().indexOf('));'))
|
||||
// def arr1 = str1.split(",")
|
||||
// println arr1.size()
|
||||
// def arr3 = str3.split(",")
|
||||
// println arr3.size()
|
||||
// var bToObj= JSON.parse(str2);
|
||||
// def dataObj = JSONObject.fromObject(json)
|
||||
// println "dd"
|
||||
}
|
||||
|
||||
def getCityMap(url, driver){
|
||||
def doc = getDocument(url, driver)
|
||||
def str = doc.toString().substring(doc.toString().indexOf("CitySearch({"), doc.toString().indexOf('<div class="g-d-dialog"'))
|
||||
String line1 = StringEscapeUtils.unescapeJava(str);
|
||||
String line2 = line1.substring(line1.indexOf("CitySearch(")+11, line1.indexOf("});")).replace("data",'"data"').replace(");","")+"}"
|
||||
def dataObj = JSONObject.fromObject(line2)
|
||||
def map = dataObj.get("data")
|
||||
def cityMap = [:]
|
||||
if(map){
|
||||
map.each {k,m->
|
||||
cityMap.put(m.city_name, m)
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
@@ -53,8 +313,9 @@ class AnjukeAppCrawl {
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
Thread.sleep(3000)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("验证码-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
@@ -107,17 +368,17 @@ class AnjukeAppCrawl {
|
||||
//"꾑�":4071,
|
||||
// def map = ["쐈갛":4070,"뮬짜":4068,"츙븜":4073,"憩쉭":4069,"르헌":16378,"낀있":15880,"쯩棺":4072,"젯쉭":16380,"틱繭":16387,"쭈都":16382,"湛揭":16392,"츙헌":16384]
|
||||
// def regionMap = ["팻땜":7,"蓍契":11, "漑쉭":15,"괜�":13,"셰땍":14,"其삠":5,"행팻":19,"쓰갛":2,"팹顧":10,"煖팻":9,"룽鳩":17,"뼝팻":4,"븀왯":8,"낀퀼":6,"쏜�":18,"녕츠":20]
|
||||
def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
|
||||
def cityMap = ["北京":regionMap]
|
||||
def cityId = 14
|
||||
// def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
|
||||
// def cityMap = ["北京":regionMap]
|
||||
def regionMap = ["铁东":1604]
|
||||
def cityId = 107
|
||||
regionMap.each{region, area_id->
|
||||
for(def i=1;i<40;i++){
|
||||
def stop = false;
|
||||
def all_stop = false
|
||||
def error_count=0
|
||||
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
|
||||
//城口
|
||||
def flagMap = getInfo(stop, i, area_id, cityId, "上海")
|
||||
while(!stop){
|
||||
def flagMap = getInfo(stop, i, area_id, cityId, "鞍山","https://m.anjuke.com/bj/community/")
|
||||
stop = flagMap.stop
|
||||
all_stop = flagMap.all_stop
|
||||
error_count++
|
||||
@@ -133,24 +394,49 @@ class AnjukeAppCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def getInfo(def stop, def i, def area_id, def cityId, def city){
|
||||
def getDetailByRegion(city, cityId, area_id, referer){
|
||||
for(def i=1;i<40;i++){
|
||||
def stop = false;
|
||||
def all_stop = false
|
||||
def error_count=0
|
||||
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
|
||||
//城口
|
||||
def flagMap = getInfo(stop, i, area_id.toString().toDouble().toInteger(), cityId.toString().toDouble().toInteger(), city, referer)
|
||||
stop = flagMap.stop
|
||||
all_stop = flagMap.all_stop
|
||||
error_count++
|
||||
if(error_count>=20){
|
||||
stop = true
|
||||
all_stop = true
|
||||
}
|
||||
}
|
||||
if(all_stop){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getInfo(def stop, def i, def area_id, def cityId, def city, def referer){
|
||||
|
||||
def map = [:]
|
||||
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
|
||||
|
||||
def header = [:]
|
||||
header.put("accept","application/json, text/plain, */*")
|
||||
header.put("accept-encoding","gzip, deflate, br")
|
||||
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||
header.put("referer","https://m.anjuke.com/sh/community/")
|
||||
header.put("sec-fetch-dest","empty")
|
||||
header.put("sec-fetch-mode","cors")
|
||||
header.put("sec-fetch-site","same-origin")
|
||||
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||
// def header = [:]
|
||||
// header.put("accept","application/json, text/plain, */*")
|
||||
// header.put("accept-encoding","gzip, deflate, br")
|
||||
// header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
// header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||
// header.put("referer",referer)
|
||||
// header.put("sec-fetch-dest","empty")
|
||||
// header.put("sec-fetch-mode","cors")
|
||||
// header.put("sec-fetch-site","same-origin")
|
||||
// header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||
// header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||
// header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||
|
||||
//https://m.anjuke.com/esf-ajax/community/list?city_id=107&page_size=100&page=1&area_id=1604&is_tw=1
|
||||
//https://m.anjuke.com/esf-ajax/community/list?city_id=107&page_size=100&page=1&area_id=1604&is_tw=1
|
||||
//https://m.anjuke.com/esf-ajax/community/list?city_id=14&page_size=100&page=1&area_id=642&is_tw=1
|
||||
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
|
||||
def all_stop = false;
|
||||
def error_count = 0
|
||||
@@ -161,7 +447,7 @@ class AnjukeAppCrawl {
|
||||
if(data && data.total.toInteger()>0){
|
||||
def communities = data.communities
|
||||
|
||||
println "【${i}】 "+communities.size() +" "+url
|
||||
println "【${i}】-${data.total} "+communities.size() +" "+url
|
||||
if(communities.size()>0){
|
||||
save(city, communities)
|
||||
Thread.sleep(10000)
|
||||
@@ -203,9 +489,20 @@ class AnjukeAppCrawl {
|
||||
udate.put("city", city)
|
||||
udate.put("c_date", new Date())
|
||||
detail.save(udate)
|
||||
println index+" [保存]"+o.jumpAction
|
||||
println index+" [保存][${city}]"+o.jumpAction
|
||||
}else{
|
||||
println index+" [已存在]"+o.jumpAction
|
||||
def udate = new BasicDBObject()
|
||||
o.each{k,v->
|
||||
udate.put(k,v)
|
||||
}
|
||||
udate.put("city", city)
|
||||
udate.put("m_date", new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("jumpAction", o.jumpAction)
|
||||
detail.update(q,new BasicDBObject('$set', udate), false, true)
|
||||
println index+" [已存在][${city}]"+o.jumpAction
|
||||
}
|
||||
|
||||
}
|
||||
@@ -219,7 +516,8 @@ class AnjukeAppCrawl {
|
||||
|
||||
def addThirdPrice(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = detail.find(q)
|
||||
// q.put("city","福州")
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -254,4 +552,29 @@ class AnjukeAppCrawl {
|
||||
q.put("s_date", SHORT_DATE)
|
||||
return thirdPartyColl.findOne(q);
|
||||
}
|
||||
|
||||
public static String ascii2Native(String str) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int begin = 0;
|
||||
int index = str.indexOf(PREFIX_UNICODE);
|
||||
try{
|
||||
while (index != -1) {
|
||||
sb.append(str.substring(begin, index));
|
||||
sb.append(ascii2Char(str.substring(index, index + 6)));
|
||||
begin = index + 6;
|
||||
index = str.indexOf(PREFIX_UNICODE, begin);
|
||||
}
|
||||
sb.append(str.substring(begin));while (index != -1) {
|
||||
sb.append(str.substring(begin, index));
|
||||
sb.append(ascii2Char(str.substring(index, index + 6)));
|
||||
begin = index + 6;
|
||||
index = str.indexOf(PREFIX_UNICODE, begin);
|
||||
}
|
||||
sb.append(str.substring(begin));
|
||||
}catch (Exception e){
|
||||
println "error: "+e.getMessage()
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+45
-9
@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.gj.abuyun.ClaABuYun
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import org.jsoup.Jsoup
|
||||
@@ -19,18 +20,26 @@ class AnjukeHouseCrawl {
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
for(def i=24;i<50;i++){
|
||||
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
|
||||
def doc = getDocument(url, driver)
|
||||
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
println "页数:"+i+" 记录数:"+eles.size()
|
||||
if(eles){
|
||||
parser(eles, driver)
|
||||
}
|
||||
}
|
||||
|
||||
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "district_detail_fang")
|
||||
|
||||
// for(def i=1;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
|
||||
// def doc = getDocument(url, driver)
|
||||
// def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
// println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles.size()
|
||||
// if(eles){
|
||||
// parser(eles, driver)
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
def parser(eles, driver){
|
||||
@@ -60,6 +69,19 @@ class AnjukeHouseCrawl {
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
|
||||
def sendMq(href, doc, project){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", project)
|
||||
json.put("url", href)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", doc.toString())
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
}
|
||||
|
||||
def parserHtml(doc) {
|
||||
String city = null
|
||||
def result = [:]
|
||||
@@ -169,6 +191,20 @@ class AnjukeHouseCrawl {
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".props-main", doc)
|
||||
}
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".house-base-info", doc)
|
||||
}
|
||||
if(!els){
|
||||
els = JQueryUtils.select("#houseMessage", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".tab-cont-right", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".village_info", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class AnjukeSendMq {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_test", "192.168.0.235", 27018)
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeSendMq();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = collection.find(q).limit(30)
|
||||
for(def d : dbcursor){
|
||||
sendMq(d.url, d.content, "community_anjuke")
|
||||
}
|
||||
}
|
||||
|
||||
def sendMq(href, doc, project){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", project)
|
||||
json.put("url", href)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", doc.toString())
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -23,14 +23,14 @@ class ZhugeFangDistCrawl {
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=11;i++){
|
||||
for(def i=1;i<=4;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
|
||||
+6
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
@@ -100,6 +101,11 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def saveToEsDistrict(def data){
|
||||
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
|
||||
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/community", jsonStr)
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
|
||||
+4
-1
@@ -58,7 +58,10 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
|
||||
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
|
||||
name = name.replace("?","").replace("&","").
|
||||
replace("·","").replace(".","").replace("(","").
|
||||
replace(")","").replace("(","").replace(")","").
|
||||
replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
|
||||
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
|
||||
if(name && name.contains("区")){
|
||||
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
|
||||
|
||||
+120
-1
@@ -3,16 +3,63 @@ package com.fangjia.script.gj.location
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
class DistrictMapMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictMapMatch()
|
||||
task.mapMatch()
|
||||
// task.mapMatch()
|
||||
// task.mapMatch1()
|
||||
// task.distince()
|
||||
|
||||
task.mapMatch2()
|
||||
}
|
||||
|
||||
def distince(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\comm_poi_map(重庆).xlsx",{ d->
|
||||
if(d['O'] && d['T'] && d['F']){
|
||||
def distance = GeoUtils.calculateDistance(d['G'].toDouble(), d['F'].toDouble(), d['T'].toDouble(), d['U'].toDouble())
|
||||
d['X'] = distance
|
||||
}
|
||||
});
|
||||
}
|
||||
def mapMatch1(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\comm_poi_map(重庆).xlsx",{ d->
|
||||
if(!d['O']){
|
||||
index++
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['B'],d['C'],d['D'])
|
||||
def map = findStandardObject2(d['B'],d['C'],d['D'], d['E'], rets);
|
||||
// if (!map || !map.lat) {
|
||||
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// }
|
||||
if(map){
|
||||
d['O'] = map.id
|
||||
if(map.adname){
|
||||
d['P'] = map.adname
|
||||
}else if(map.region){
|
||||
d['P'] = map.region
|
||||
}
|
||||
d['Q'] = map.name
|
||||
d['R'] = map.address
|
||||
if(map.type){
|
||||
d['S'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['S'] = map.estateType
|
||||
}
|
||||
d['T'] = map.lng
|
||||
d['U'] = map.lat
|
||||
d['V'] = map.score
|
||||
d['W'] = map.source
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
@@ -49,6 +96,49 @@ class DistrictMapMatch {
|
||||
}
|
||||
|
||||
|
||||
def mapMatch2(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\待入库小区(1).xlsx",{ d->
|
||||
if(!d['M']){
|
||||
index++
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['C'],d['J'],d['E'])
|
||||
def map = findStandardObject2(d['C'],d['J'],d['E'], d['A'], rets);
|
||||
// if (!map || !map.lat) {
|
||||
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// }
|
||||
if(map){
|
||||
d['K'] = map.id
|
||||
if(map.adname){
|
||||
d['L'] = map.adname
|
||||
}else if(map.region){
|
||||
d['L'] = map.region
|
||||
}
|
||||
d['M'] = map.name
|
||||
d['N'] = map.address
|
||||
|
||||
d['O'] = map.lng
|
||||
d['P'] = map.lat
|
||||
if(map.type){
|
||||
d['Q'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['Q'] = map.estateType
|
||||
}
|
||||
d['R'] = map.score
|
||||
|
||||
if(d['O'] && d['AM']){
|
||||
def location = d['AM']
|
||||
def lng = location.toString().substring(location.toString().indexOf(": ")+2, location.toString().indexOf(","))
|
||||
def lat = location.toString().substring(location.toString().lastIndexOf(": ")+2, location.toString().indexOf("}"))
|
||||
def distance = GeoUtils.calculateDistance(d['O'].toDouble(), d['P'].toDouble(), lng.toDouble(), lat.toDouble())
|
||||
d['S'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
@@ -118,6 +208,35 @@ class DistrictMapMatch {
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getLocationInfo(def city, def address){
|
||||
def gloJson = GaoDeApiUtils.geo(city, address)
|
||||
if(gloJson) {
|
||||
def glocation = null
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
glocation = geocodes.get(0).location
|
||||
}
|
||||
return glocation
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getStreetInfo(def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
}
|
||||
}
|
||||
|
||||
+11
-9
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -216,7 +216,8 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detailUrl)
|
||||
detail.url = detailUrl
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
@@ -229,7 +230,7 @@ public class BeiKeCrawlByChrome {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
@@ -309,7 +310,8 @@ public class BeiKeCrawlByChrome {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
detail.url = detailUrl
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
@@ -326,7 +328,7 @@ public class BeiKeCrawlByChrome {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
@@ -337,15 +339,15 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detailUrl){
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
// q.put("city", detail.city)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
q.put("url", detailUrl)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("url", detail.url)
|
||||
// q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
|
||||
@@ -308,4 +308,5 @@ class DataUtil {
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+12
-10
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
task.standardRegion()
|
||||
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-03-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
@@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -300,6 +300,7 @@ public class LianjiaCrawlByChrome {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "Á´¼ÒÔÚÏß"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
@@ -313,7 +314,7 @@ public class LianjiaCrawlByChrome {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
@@ -423,12 +424,13 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
// q.put("city", detail.city)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
// q.put("trade_date", detail.trade_date)
|
||||
q.put("url", detail.url)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
|
||||
+53
-5
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -30,22 +31,69 @@ class PaimaiTest {
|
||||
def task = new PaimaiTest();
|
||||
// task.run();
|
||||
// task.stat()
|
||||
task.updatePaimai()
|
||||
// task.updatePaimai()
|
||||
// task.updateFieldPaimai()
|
||||
// task.statCity()
|
||||
task.jsStat()
|
||||
}
|
||||
|
||||
def jsStat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","苏州")
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
def m = [:]
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def statCity(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
def dbcursor = paimai43_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
if(index%10000==0){
|
||||
print "."
|
||||
}
|
||||
if(!set.contains(d.city)){
|
||||
set << d.city
|
||||
}
|
||||
}
|
||||
|
||||
println set.size()
|
||||
for(def s : set){
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def updateFieldPaimai(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
|
||||
// q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
|
||||
def index = 0
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def obj : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(obj.address && obj.address.contains("可贷款")){
|
||||
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
def oldCity = obj.city
|
||||
if(oldCity){
|
||||
def newCity = RegionData.formatCity(oldCity)
|
||||
if(newCity && newCity!=oldCity){
|
||||
obj.put("city", newCity)
|
||||
println index +" "+ oldCity +" "+newCity
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
|
||||
}
|
||||
}
|
||||
// if(obj.address && obj.address.contains("可贷款")){
|
||||
// obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
|
||||
// paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.fangjia.script.temp.shenjian.oracle;
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DistrictCodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* 导出异常价格
|
||||
*/
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sql = "SELECT id, cityname, distname, commid, commname, calc_type, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('202206', '202205') " +
|
||||
" AND cityname IN ('东莞', '上海') AND rownum <= 10 ORDER BY cityname, distname, commid, commname, year_month"
|
||||
def result = []
|
||||
result << ["城市", "区域", "小区ID", "小区名", "五月均价", "五月计算方式", "六月均价", "六月计算方式"]
|
||||
Map<String, Integer> fiveMap = new HashMap<>()
|
||||
Map<String, Integer> sixMap = new HashMap<>()
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def year_month = r['year_month']
|
||||
String key = r['cityname'] + "," + r['distname'] + "," + r['commid'] + "," + r['commname']
|
||||
if (year_month == '202205') {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
} else if ((year_month == '202206')) {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
}
|
||||
})
|
||||
for (String key : sixMap.keySet()) {
|
||||
Integer fivePrice = fiveMap.get(key).split("|")[0]
|
||||
Integer sixPrice = sixMap.get(key).split("|")[0]
|
||||
if (fivePrice != null && sixPrice != null) {
|
||||
if (Math.abs(sixPrice - fivePrice) * 100 / fivePrice > 10) {
|
||||
def info = key.split(",")
|
||||
result << [info[0], info[1], info[2], info[3], fivePrice, fiveMap.get(key).split("|")[1], sixPrice, sixMap.get(key).split("|")[1]]
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
+6
@@ -26,6 +26,8 @@ class CrawalDBUtil {
|
||||
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
|
||||
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
|
||||
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
static DBCollection arroundTempCol = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "arround_temp")
|
||||
|
||||
|
||||
public static getSeedsCol() {
|
||||
return seedsColl;
|
||||
@@ -38,6 +40,10 @@ class CrawalDBUtil {
|
||||
return thirdPartyColl;
|
||||
}
|
||||
|
||||
public static getArroundTempCol() {
|
||||
return arroundTempCol;
|
||||
}
|
||||
|
||||
public static getDistrictImageColl(){
|
||||
return imagesColl;
|
||||
}
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "58ͬ³Ç"//
|
||||
def citys = findCrawlCity(source, 202202);
|
||||
def source = "Q·¿"//
|
||||
def citys = findCrawlCity(source, 202204);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202203);
|
||||
def count2 = findCount(source, city, 202204);
|
||||
def count1 = findCount(source, city, 202204);
|
||||
def count2 = findCount(source, city, 202203);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+4
-4
@@ -193,7 +193,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州",
|
||||
def citys = ["鉗�","揭갛","躬捺","憩鹵","麗쉭","陵뇽","쉭쳔","쐽鹵","굇쑴","뷜捺","켓쑴","깝鹵","蠟소왯","�헴","머鹵","빤鹵","갛헤","좔냘","讀鏤","막捺","鮫갛","交빤","섈주","件퀼","같�","땜檀","갛捺","驅捺"]
|
||||
def citys = ["宁波","大连","佛山","乌鲁木齐","济南","长春","兰州","石家庄","温州","烟台","太原","泉州","南通","呼和浩特","徐州","惠州","厦门","南昌","临沂","贵阳","嘉兴","常州","盐城","威海","淄博","深圳","中山","重庆","苏州","西宁","绍兴","洛阳","廊坊","昆明","金华","珠海","宜昌","南充","信阳","杭州","沧州","宿迁","南阳","德州","桂林","新乡","包头","遵义","乐山","株洲","德阳","眉山","驻马店","许昌","宜宾","衡水","泸州","营口","清远","达州","宝鸡","长沙","广安"]
|
||||
for(def city : citys){
|
||||
def str = cityList.get(city)
|
||||
for(def i=0;i<1;i++){
|
||||
@@ -207,10 +207,10 @@ class Esf58Crawl extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cityList.remove("福州")
|
||||
// cityList.each{city, str->
|
||||
// for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
//// if(citys.contains(city)){
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
@@ -220,7 +220,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// }
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
+49
-6
@@ -2,6 +2,7 @@ package com.fangjia.script.thirdPriceCrawl
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
import org.bson.types.ObjectId
|
||||
@@ -15,18 +16,54 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
//"fang.com":"搜房",
|
||||
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
def esSourceMap = ["ke.com":"贝壳"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
|
||||
esSourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
def rets = findAllEsData(city, v)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, "贝壳找房")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(city,source){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='贝壳' and city='${city}'", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
def data = new BasicDBObject()
|
||||
data.put("city", d.city)
|
||||
data.put("region", d.region)
|
||||
data.put("name", d.name)
|
||||
data.put("address", d.address)
|
||||
data.put("avg_price", d.avg_price)
|
||||
data.put("source", source)
|
||||
data.put("s_date", SHORT_DATE)
|
||||
rets << data
|
||||
}
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["重庆"]//CommonUtils.findAllCity()
|
||||
def citys = CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
@@ -95,8 +132,14 @@ class SynchDistrictPrice {
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(obj){
|
||||
updateIndex++
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
thirdPartyColl.save(r)
|
||||
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
// thirdPartyColl.save(r)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("avg_price", r.avg_price)
|
||||
udata.put("m_date",new Date())
|
||||
thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata)
|
||||
|
||||
}else{
|
||||
try{
|
||||
addIndex++
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202203);
|
||||
def count2 = findTotalCount(source, 202204);
|
||||
def count1 = findTotalCount(source, 202204);
|
||||
def count2 = findTotalCount(source, 202205);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
q.put("s_date", SHORT_DATE);
|
||||
def ret = []
|
||||
try{
|
||||
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type", 1))
|
||||
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1))
|
||||
|
||||
if(dbcursor){
|
||||
for(def dbo : dbcursor){
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220501", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220601", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 539 B After Width: | Height: | Size: 269 B |
Binary file not shown.
|
After Width: | Height: | Size: 324 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 123 KiB |
Reference in New Issue
Block a user